Index: include/llvm/CodeGen/GlobalISel/InstructionSelector.h =================================================================== --- include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -17,6 +17,7 @@ #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTOR_H #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Optional.h" #include #include #include @@ -207,6 +208,11 @@ /// - InsnID - Instruction ID to modify /// - RendererID - The renderer to call GIR_ComplexRenderer, + /// Render sub-operands of complex operands to the specified instruction + /// - InsnID - Instruction ID to modify + /// - RendererID - The renderer to call + /// - RenderOpID - The suboperand to render. + GIR_ComplexSubOperandRenderer, /// Render a G_CONSTANT operator as a sign-extended immediate. /// - NewInsnID - Instruction ID to modify @@ -265,12 +271,13 @@ virtual bool select(MachineInstr &I) const = 0; protected: - using ComplexRendererFn = std::function; + using ComplexRendererFn = + Optional, 4>>; using RecordedMIVector = SmallVector; using NewMIVector = SmallVector; struct MatcherState { - std::vector Renderers; + std::vector Renderers; RecordedMIVector MIs; MatcherState(unsigned MaxRenderers); Index: include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h =================================================================== --- include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -273,12 +273,14 @@ << "), ComplexPredicateID=" << ComplexPredicateID << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); // FIXME: Use std::invoke() when it's available. - if (!(State.Renderers[RendererID] = - (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( - State.MIs[InsnID]->getOperand(OpIdx)))) { + ComplexRendererFn Renderer = + (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( + State.MIs[InsnID]->getOperand(OpIdx)); + if (Renderer.hasValue()) + State.Renderers[RendererID] = Renderer.getValue(); + else if (handleReject() == RejectAndGiveUp) return false; - } break; } @@ -475,11 +477,23 @@ int64_t InsnID = MatchTable[CurrentIdx++]; int64_t RendererID = MatchTable[CurrentIdx++]; assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); - State.Renderers[RendererID](OutMIs[InsnID]); + for (const auto &RenderOpFn : State.Renderers[RendererID]) + RenderOpFn(OutMIs[InsnID]); DEBUG(dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" << InsnID << "], " << RendererID << ")\n"); break; } + case GIR_ComplexSubOperandRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t RendererID = MatchTable[CurrentIdx++]; + int64_t RenderOpID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); + DEBUG(dbgs() << CurrentIdx << ": GIR_ComplexSubOperandRenderer(OutMIs[" + << InsnID << "], " << RendererID << ", " << RenderOpID + << ")\n"); + break; + } case GIR_CopyConstantAsSImm: { int64_t NewInsnID = MatchTable[CurrentIdx++]; Index: lib/CodeGen/GlobalISel/InstructionSelector.cpp =================================================================== --- lib/CodeGen/GlobalISel/InstructionSelector.cpp +++ lib/CodeGen/GlobalISel/InstructionSelector.cpp @@ -29,7 +29,7 @@ using namespace llvm; InstructionSelector::MatcherState::MatcherState(unsigned MaxRenderers) - : Renderers(MaxRenderers, nullptr), MIs() {} + : Renderers(MaxRenderers, {}), MIs() {} InstructionSelector::InstructionSelector() = default; Index: lib/Target/AArch64/AArch64InstructionSelector.cpp =================================================================== --- lib/Target/AArch64/AArch64InstructionSelector.cpp +++ lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -1366,13 +1366,13 @@ else if (Root.isReg()) { MachineInstr *Def = MRI.getVRegDef(Root.getReg()); if (Def->getOpcode() != TargetOpcode::G_CONSTANT) - return nullptr; + return None; MachineOperand &Op1 = Def->getOperand(1); if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64) - return nullptr; + return None; Immed = Op1.getCImm()->getZExtValue(); } else - return nullptr; + return None; unsigned ShiftAmt; @@ -1382,10 +1382,13 @@ ShiftAmt = 12; Immed = Immed >> 12; } else - return nullptr; + return None; unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt); - return [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed).addImm(ShVal); }; + return {{ + [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); }, + [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); }, + }}; } namespace llvm { Index: test/TableGen/GlobalISelEmitter.td =================================================================== --- test/TableGen/GlobalISelEmitter.td +++ test/TableGen/GlobalISelEmitter.td @@ -31,6 +31,12 @@ def gi_complex : GIComplexOperandMatcher, GIComplexPatternEquiv; +def complex_rr : Operand, ComplexPattern { + let MIOperandInfo = (ops GPR32, GPR32); +} +def gi_complex_rr : + GIComplexOperandMatcher, + GIComplexPatternEquiv; def m1 : OperandWithDefaultOps ; def Z : OperandWithDefaultOps ; @@ -56,6 +62,7 @@ // CHECK-NEXT: MatcherInfo({TypeObjects, FeatureBitsets, I64ImmPredicateFns, APIntImmPredicateFns, APFloatImmPredicateFns, { // CHECK-NEXT: nullptr, // GICP_Invalid // CHECK-NEXT: &MyTargetInstructionSelector::selectComplexPattern, // gi_complex +// CHECK-NEXT: &MyTargetInstructionSelector::selectComplexPatternRR, // gi_complex_rr // CHECK-NEXT: }}) // CHECK-NEXT: #endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT @@ -107,6 +114,7 @@ // CHECK-NEXT: enum { // CHECK-NEXT: GICP_Invalid, // CHECK-NEXT: GICP_gi_complex, +// CHECK-NEXT: GICP_gi_complex_rr, // CHECK-NEXT: }; // CHECK-LABEL: // PatFrag predicates. @@ -180,9 +188,13 @@ // CHECK-NEXT: // Label 0: @[[LABEL]] def INSN3 : I<(outs GPR32:$dst), - (ins GPR32Op:$src1, complex:$src2, GPR32:$src3, complex:$src4, complex:$src5), []>; -def : Pat<(select GPR32:$src1, complex:$src2, (select GPR32:$src3, complex:$src4, complex:$src5)), - (INSN3 GPR32:$src1, complex:$src2, GPR32:$src3, complex:$src4, complex:$src5)>; + (ins GPR32Op:$src1, GPR32:$src2a, GPR32:$src2b, GPR32:$src3, complex:$src4, i32imm:$src5a, i32imm:$src5b), []>; +def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), + (select GPR32:$src3, + complex:$src4, + (complex i32imm:$src5a, i32imm:$src5b))), + (INSN3 GPR32:$src1, GPR32:$src2b, GPR32:$src2a, GPR32:$src3, + complex:$src4, i32imm:$src5a, i32imm:$src5b)>; //===- Test a pattern with multiple ComplexPattern operands. --------------===// // @@ -198,9 +210,9 @@ // CHECK-NEXT: // MIs[0] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] src2 +// CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex, +// CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex_rr, // CHECK-NEXT: // MIs[0] Operand 3 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SELECT, @@ -212,18 +224,20 @@ // CHECK-NEXT: // MIs[1] src4 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/1, /*Op*/2, /*Renderer*/1, GICP_gi_complex, -// CHECK-NEXT: // MIs[1] src5 +// CHECK-NEXT: // MIs[1] Operand 3 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/2, GICP_gi_complex, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, (select:{ *:[i32] } GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, complex:{ *:[i32] }:$src5)) => (INSN3:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, complex:{ *:[i32] }:$src5) +// CHECK-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, (complex_rr:{ *:[i32] } GPR32:{ *:[i32] }:$src2a, GPR32:{ *:[i32] }:$src2b), (select:{ *:[i32] } GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, (complex:{ *:[i32] } i32imm:{ *:[i32] }:$src5a, i32imm:{ *:[i32] }:$src5b))) => (INSN3:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2b, GPR32:{ *:[i32] }:$src2a, GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, i32imm:{ *:[i32] }:$src5a, i32imm:{ *:[i32] }:$src5b) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN3, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 -// CHECK-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0, +// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // src2b +// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src2a // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src3 // CHECK-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/1, -// CHECK-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/2, +// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/2, /*SubOperand*/0, // src5a +// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/2, /*SubOperand*/1, // src5b // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // CHECK-NEXT: GIR_Done, Index: utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- utils/TableGen/GlobalISelEmitter.cpp +++ utils/TableGen/GlobalISelEmitter.cpp @@ -197,7 +197,12 @@ if (Operator->isSubClassOf("Intrinsic")) return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str(); - return " (Operator not understood)"; + if (Operator->isSubClassOf("ComplexPattern")) + return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() + + ")") + .str(); + + return (" (Operator " + Operator->getName() + " not understood)").str(); } /// Helper function to let the emitter report skip reason error messages. @@ -508,10 +513,17 @@ ArrayRef SrcLoc; + typedef std::tuple + DefinedComplexPatternSubOperand; + typedef StringMap + DefinedComplexPatternSubOperandMap; + /// A map of Symbolic Names to ComplexPattern sub-operands. + DefinedComplexPatternSubOperandMap ComplexSubOperands; + public: RuleMatcher(ArrayRef SrcLoc) : Matchers(), Actions(), InsnVariableIDs(), DefinedOperands(), - NextInsnVarID(0), SrcLoc(SrcLoc) {} + NextInsnVarID(0), SrcLoc(SrcLoc), ComplexSubOperands() {} RuleMatcher(RuleMatcher &&Other) = default; RuleMatcher &operator=(RuleMatcher &&Other) = default; @@ -541,6 +553,20 @@ void defineOperand(StringRef SymbolicName, OperandMatcher &OM); + void defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern, + unsigned RendererID, unsigned SubOperandID) { + assert(ComplexSubOperands.count(SymbolicName) == 0 && "Already defined"); + ComplexSubOperands[SymbolicName] = + std::make_tuple(ComplexPattern, RendererID, SubOperandID); + } + Optional + getComplexSubOperand(StringRef SymbolicName) const { + const auto &I = ComplexSubOperands.find(SymbolicName); + if (I == ComplexSubOperands.end()) + return None; + return I->second; + } + const InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const; const OperandMatcher &getOperandMatcher(StringRef Name) const; @@ -1522,6 +1548,9 @@ /// The renderer number. This must be unique within a rule since it's used to /// identify a temporary variable to hold the renderer function. unsigned RendererID; + /// When provided, this is the suboperand of the ComplexPattern operand to + /// render. Otherwise all the suboperands will be rendered. + Optional SubOperand; unsigned getNumOperands() const { return TheDef.getValueAsDag("Operands")->getNumArgs(); @@ -1529,19 +1558,26 @@ public: RenderComplexPatternOperand(unsigned InsnID, const Record &TheDef, - StringRef SymbolicName, unsigned RendererID) + StringRef SymbolicName, unsigned RendererID, + Optional SubOperand = None) : OperandRenderer(OR_ComplexPattern), InsnID(InsnID), TheDef(TheDef), - SymbolicName(SymbolicName), RendererID(RendererID) {} + SymbolicName(SymbolicName), RendererID(RendererID), + SubOperand(SubOperand) {} static bool classof(const OperandRenderer *R) { return R->getKind() == OR_ComplexPattern; } void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { - Table << MatchTable::Opcode("GIR_ComplexRenderer") + Table << MatchTable::Opcode(SubOperand.hasValue() ? "GIR_ComplexSubOperandRenderer" + : "GIR_ComplexRenderer") << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) << MatchTable::Comment("RendererID") - << MatchTable::IntValue(RendererID) << MatchTable::LineBreak; + << MatchTable::IntValue(RendererID); + if (SubOperand.hasValue()) + Table << MatchTable::Comment("SubOperand") + << MatchTable::IntValue(SubOperand.getValue()); + Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; } }; @@ -2007,11 +2043,12 @@ Record *findNodeEquiv(Record *N) const; Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); - Expected - createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, - const TreePatternNode *Src, - unsigned &TempOpIdx) const; - Error importChildMatcher(InstructionMatcher &InsnMatcher, + Expected createAndImportSelDAGMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx) const; + Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R, + unsigned &TempOpIdx) const; + Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher, const TreePatternNode *SrcChild, unsigned OpIdx, unsigned &TempOpIdx) const; Expected @@ -2075,10 +2112,9 @@ return Error::success(); } -Expected -GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, - const TreePatternNode *Src, - unsigned &TempOpIdx) const { +Expected GlobalISelEmitter::createAndImportSelDAGMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx) const { Record *SrcGIEquivOrNull = nullptr; const CodeGenInstruction *SrcGIOrNull = nullptr; @@ -2177,8 +2213,8 @@ return failedImport("Expected IntInit containing instrinsic ID)"); } - if (auto Error = - importChildMatcher(InsnMatcher, SrcChild, OpIdx++, TempOpIdx)) + if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild, OpIdx++, + TempOpIdx)) return std::move(Error); } } @@ -2186,7 +2222,20 @@ return InsnMatcher; } -Error GlobalISelEmitter::importChildMatcher(InstructionMatcher &InsnMatcher, +Error GlobalISelEmitter::importComplexPatternOperandMatcher( + OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const { + const auto &ComplexPattern = ComplexPatternEquivs.find(R); + if (ComplexPattern == ComplexPatternEquivs.end()) + return failedImport("SelectionDAG ComplexPattern (" + R->getName() + + ") not mapped to GlobalISel"); + + OM.addPredicate(OM, *ComplexPattern->second); + TempOpIdx++; + return Error::success(); +} + +Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, + InstructionMatcher &InsnMatcher, const TreePatternNode *SrcChild, unsigned OpIdx, unsigned &TempOpIdx) const { @@ -2219,12 +2268,32 @@ // Check for nested instructions. if (!SrcChild->isLeaf()) { + if (SrcChild->getOperator()->isSubClassOf("ComplexPattern")) { + // When a ComplexPattern is used as an operator, it should do the same + // thing as when used as a leaf. However, the children of the operator + // name the sub-operands that make up the complex operand and we must + // prepare to reference them in the renderer too. + unsigned RendererID = TempOpIdx; + if (auto Error = importComplexPatternOperandMatcher( + OM, SrcChild->getOperator(), TempOpIdx)) + return Error; + + for (unsigned i = 0, e = SrcChild->getNumChildren(); i != e; ++i) { + auto *SubOperand = SrcChild->getChild(i); + if (!SubOperand->getName().empty()) + Rule.defineComplexSubOperand(SubOperand->getName(), + SrcChild->getOperator(), RendererID, i); + } + + return Error::success(); + } + // Map the node to a gMIR instruction. InstructionOperandMatcher &InsnOperand = OM.addPredicate(InsnMatcher.getRuleMatcher(), SrcChild->getName()); auto InsnMatcherOrError = createAndImportSelDAGMatcher( - InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); + Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); if (auto Error = InsnMatcherOrError.takeError()) return Error; @@ -2257,17 +2326,8 @@ } // Check for ComplexPattern's. - if (ChildRec->isSubClassOf("ComplexPattern")) { - const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); - if (ComplexPattern == ComplexPatternEquivs.end()) - return failedImport("SelectionDAG ComplexPattern (" + - ChildRec->getName() + ") not mapped to GlobalISel"); - - OM.addPredicate(OM, - *ComplexPattern->second); - TempOpIdx++; - return Error::success(); - } + if (ChildRec->isSubClassOf("ComplexPattern")) + return importComplexPatternOperandMatcher(OM, ChildRec, TempOpIdx); if (ChildRec->isSubClassOf("ImmLeaf")) { return failedImport( @@ -2289,6 +2349,14 @@ DstChild->getTransformFn()->getName()); } + const auto &SubOperand = Rule.getComplexSubOperand(DstChild->getName()); + if (SubOperand.hasValue()) { + DstMIBuilder.addRenderer( + 0, *std::get<0>(*SubOperand), DstChild->getName(), + std::get<1>(*SubOperand), std::get<2>(*SubOperand)); + return Error::success(); + } + if (!DstChild->isLeaf()) { // We accept 'bb' here. It's an operator because BasicBlockSDNode isn't // inline, but in MI it's just another operand. @@ -2526,7 +2594,7 @@ InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName()); unsigned TempOpIdx = 0; auto InsnMatcherOrError = - createAndImportSelDAGMatcher(InsnMatcherTemp, Src, TempOpIdx); + createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx); if (auto Error = InsnMatcherOrError.takeError()) return std::move(Error); InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();