Index: llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -113,6 +113,12 @@ /// - Expected opcode GIM_CheckOpcode, + /// Check the opcode on the specified instruction, if instruction was copy + /// check the opcode of the instruction that was source of the copy. + /// - InsnID - Instruction ID + /// - Expected opcode + GIM_CheckOpcodeIgnoreCopies, + /// Check the opcode on the specified instruction, checking 2 acceptable /// alternatives. /// - InsnID - Instruction ID Index: llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -155,6 +155,7 @@ } case GIM_CheckOpcode: + case GIM_CheckOpcodeIgnoreCopies: case GIM_CheckOpcodeIsEither: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t Expected0 = MatchTable[CurrentIdx++]; @@ -164,6 +165,17 @@ assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); unsigned Opcode = State.MIs[InsnID]->getOpcode(); + if (MatcherOpcode == GIM_CheckOpcodeIgnoreCopies && + Opcode == TargetOpcode::COPY) { + Register CopySrc = State.MIs[InsnID]->getOperand(1).getReg(); + if (!CopySrc.isPhysical()) { + auto MI2 = State.MIs[InsnID] = getDefIgnoringCopies(CopySrc, MRI); + if (MI2) { + State.MIs[InsnID] = MI2; + Opcode = MI2->getOpcode(); + } + } + } DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID Index: llvm/include/llvm/Target/TargetSelectionDAG.td =================================================================== --- llvm/include/llvm/Target/TargetSelectionDAG.td +++ llvm/include/llvm/Target/TargetSelectionDAG.td @@ -308,6 +308,9 @@ def implicit; def node; def srcvalue; +// Used by global-isel, looks through cross register bank copies. +// Does not increase complexity of the pattern. Ignored by sdag emitter. +def g_maybe_cross_reg_bank_copy; def imm : SDNode<"ISD::Constant" , SDTIntLeaf , [], "ConstantSDNode">; def timm : SDNode<"ISD::TargetConstant",SDTIntLeaf, [], "ConstantSDNode">; Index: llvm/lib/Target/AMDGPU/VOP3Instructions.td =================================================================== --- llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -578,7 +578,7 @@ // When the inner operation is used multiple times, selecting 3-op // instructions may still be beneficial -- if the other users can be // combined similarly. Let's be conservative for now. - (op2 (HasOneUseBinOp node:$x, node:$y), node:$z), + (op2 (g_maybe_cross_reg_bank_copy (HasOneUseBinOp node:$x, node:$y)), node:$z), [{ // Only use VALU ops when the result is divergent. if (!N->isDivergent()) Index: llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll @@ -45,9 +45,8 @@ ; ; GFX10-LABEL: add_shl_vgpr_c: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_add_i32 s2, s2, s3 +; GFX10-NEXT: v_add_lshl_u32 v0, s2, s3, v0 ; GFX10-NEXT: ; implicit-def: $vcc_hi -; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s2 ; GFX10-NEXT: ; return to shader part epilog %x = add i32 %a, %b %result = shl i32 %x, %c Index: llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir +++ llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir @@ -162,10 +162,8 @@ ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1 ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX10: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def $scc - ; GFX10: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[S_AND_B32_]] - ; GFX10: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY3]], [[COPY2]], implicit $exec - ; GFX10: S_ENDPGM 0, implicit [[V_OR_B32_e64_]] + ; GFX10: [[V_AND_OR_B32_:%[0-9]+]]:vgpr_32 = V_AND_OR_B32 [[COPY]], [[COPY1]], [[COPY2]], implicit $exec + ; GFX10: S_ENDPGM 0, implicit [[V_AND_OR_B32_]] %0:sgpr(s32) = COPY $sgpr0 %1:sgpr(s32) = COPY $sgpr1 %2:vgpr(s32) = COPY $vgpr0 Index: llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir +++ llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir @@ -126,10 +126,8 @@ ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1 ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX10: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def $scc - ; GFX10: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[S_XOR_B32_]] - ; GFX10: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY3]], [[COPY2]], implicit $exec - ; GFX10: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]] + ; GFX10: [[V_XOR3_B32_:%[0-9]+]]:vgpr_32 = V_XOR3_B32 [[COPY]], [[COPY1]], [[COPY2]], implicit $exec + ; GFX10: S_ENDPGM 0, implicit [[V_XOR3_B32_]] %0:sgpr(s32) = COPY $sgpr0 %1:sgpr(s32) = COPY $sgpr1 %2:vgpr(s32) = COPY $vgpr0 @@ -174,10 +172,8 @@ ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1 ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0 - ; GFX10: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def $scc - ; GFX10: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[S_XOR_B32_]] - ; GFX10: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY2]], [[COPY3]], implicit $exec - ; GFX10: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]] + ; GFX10: [[V_XOR3_B32_:%[0-9]+]]:vgpr_32 = V_XOR3_B32 [[COPY]], [[COPY1]], [[COPY2]], implicit $exec + ; GFX10: S_ENDPGM 0, implicit [[V_XOR3_B32_]] %0:sgpr(s32) = COPY $sgpr0 %1:sgpr(s32) = COPY $sgpr1 %2:vgpr(s32) = COPY $vgpr0 Index: llvm/test/TableGen/GlobalISelEmitterCustomPredicateIgnoreCrossRegBankCopies.td =================================================================== --- /dev/null +++ llvm/test/TableGen/GlobalISelEmitterCustomPredicateIgnoreCrossRegBankCopies.td @@ -0,0 +1,104 @@ +// RUN: llvm-tblgen %s -gen-global-isel -optimize-match-table=false -I %p/../../include -I %p/Common -o - | FileCheck %s + +include "llvm/Target/Target.td" +include "GlobalISelEmitterCommon.td" + +// Boilerplate code for setting up some registers with subregs. +class MyReg subregs = []> + : Register { + let SubRegs = subregs; +} + +class MyClass types, dag registers> + : RegisterClass<"Test", types, size, registers> { + let Size = size; +} + +def sub0 : SubRegIndex<16>; +def sub1 : SubRegIndex<16, 16>; +def S0 : MyReg<"s0">; +def S1 : MyReg<"s1">; +def SRegs : MyClass<16, [i16], (sequence "S%u", 0, 1)>; + +let SubRegIndices = [sub0, sub1] in { +def D0 : MyReg<"d0", [S0, S1]>; +} + +def DRegs : MyClass<32, [i32], (sequence "D%u", 0, 0)>; +def DOP : RegisterOperand; +def AND_OR : I<(outs DRegs:$dst), (ins DOP:$src0, DOP:$src1, DOP:$src2), []>; + +def and_or_pat : PatFrag< + (ops node:$x, node:$y, node:$z), + (and (g_maybe_cross_reg_bank_copy (or node:$x, node:$y)), node:$z), [{ return foo(); }]> { + let GISelPredicateCode = [{ + return doesComplexCheck(MI); + }]; + let PredicateCodeUsesOperands = 1; +} + +// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 1 // +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, +// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] src2 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:1:z +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] Operand 2 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1] +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, +// CHECK-NEXT: GIM_CheckOpcodeIgnoreCopies, /*MI*/1, TargetOpcode::G_OR, +// CHECK-NEXT: // MIs[1] Operand 0 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: // MIs[1] src0 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:1:x +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[1] src1 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:1:y +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, +// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, +// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:1:z, (g_maybe_cross_reg_bank_copy:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:1:x, DOP:{ *:[i32] }:$src1:$pred:1:y)))<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, + +// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 0 // +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, +// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] Operand 1 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, +// CHECK-NEXT: GIM_CheckOpcodeIgnoreCopies, /*MI*/1, TargetOpcode::G_OR, +// CHECK-NEXT: // MIs[1] Operand 0 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: // MIs[1] src0 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:1:x +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[1] src1 +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:1:y +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] src2 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:1:z +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, +// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, +// CHECK-NEXT: // (and:{ *:[i32] } (g_maybe_cross_reg_bank_copy:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:1:x, DOP:{ *:[i32] }:$src1:$pred:1:y)), DOP:{ *:[i32] }:$src2:$pred:1:z)<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, + +// Test g_maybe_cross_reg_bank_copy. +def : Pat< + (i32 (and_or_pat DOP:$src0, DOP:$src1, DOP:$src2)), + (AND_OR DOP:$src0, DOP:$src1, DOP:$src2) +>; Index: llvm/utils/TableGen/CodeGenDAGPatterns.cpp =================================================================== --- llvm/utils/TableGen/CodeGenDAGPatterns.cpp +++ llvm/utils/TableGen/CodeGenDAGPatterns.cpp @@ -1371,6 +1371,10 @@ static unsigned getPatternSize(const TreePatternNode *P, const CodeGenDAGPatterns &CGP) { unsigned Size = 3; // The node itself. + if (!P->isLeaf() && + P->getOperator()->getName().equals("g_maybe_cross_reg_bank_copy")) { + return getPatternSize(P->getChild(0), CGP); + } // If the root node is a ConstantSDNode, increases its size. // e.g. (set R32:$dst, 0). if (P->isLeaf() && isa(P->getLeafValue())) @@ -1775,6 +1779,9 @@ Operator->getName() == "implicit") return 0; // All return nothing. + if (Operator->getName() == "g_maybe_cross_reg_bank_copy") + return 1; + if (Operator->isSubClassOf("Intrinsic")) return CDP.getIntrinsic(Operator).IS.RetVTs.size(); @@ -2615,6 +2622,12 @@ return MadeChange; } + if (getOperator()->getName() == "g_maybe_cross_reg_bank_copy") { + bool MadeChange = getChild(0)->ApplyTypeConstraints(TP, NotRegisters); + MadeChange |= this->UpdateNodeType(0, getChild(0)->getExtType(0),TP); + return MadeChange; + } + assert(getOperator()->isSubClassOf("SDNodeXForm") && "Unknown node type!"); // Node transforms always take one operand. @@ -2662,6 +2675,9 @@ if (getOperator()->isSubClassOf("ComplexPattern")) return true; + if (getOperator()->getName() == "g_maybe_cross_reg_bank_copy") + return true; + // If this node is a commutative operator, check that the LHS isn't an // immediate. const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(getOperator()); @@ -2817,7 +2833,8 @@ !Operator->isSubClassOf("Intrinsic") && !Operator->isSubClassOf("ComplexPattern") && Operator->getName() != "set" && - Operator->getName() != "implicit") + Operator->getName() != "implicit" && + Operator->getName() != "g_maybe_cross_reg_bank_copy") error("Unrecognized node '" + Operator->getName() + "'!"); // Check to see if this is something that is illegal in an input pattern. @@ -4506,7 +4523,8 @@ CodeGenDAGPatterns &CDP, const MultipleUseVarSet &DepVars) { // We cannot permute leaves or ComplexPattern uses. - if (N->isLeaf() || N->getOperator()->isSubClassOf("ComplexPattern")) { + if (N->isLeaf() || N->getOperator()->isSubClassOf("ComplexPattern") || + N->getOperator()->getName() == "g_maybe_cross_reg_bank_copy") { OutVariants.push_back(N); return; } Index: llvm/utils/TableGen/DAGISelMatcherGen.cpp =================================================================== --- llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -499,6 +499,11 @@ void MatcherGen::EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes, unsigned ForceMode) { + // This operator is used by global-isel, skip it for sdag. + if (!N->isLeaf() && + N->getOperator()->getName().equals("g_maybe_cross_reg_bank_copy")) + return EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0), ForceMode); + // If N and NodeNoTypes don't agree on a type, then this is a case where we // need to do a type check. Emit the check, apply the type to NodeNoTypes and // reinfer any correlated types. Index: llvm/utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- llvm/utils/TableGen/GlobalISelEmitter.cpp +++ llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -1738,6 +1738,7 @@ static DenseMap OpcodeValues; + bool SkipCopies; MatchTableRecord getInstValue(const CodeGenInstruction *I) const { const auto VI = OpcodeValues.find(I); @@ -1757,9 +1758,10 @@ } InstructionOpcodeMatcher(unsigned InsnVarID, - ArrayRef I) + ArrayRef I, + bool SkipCopies = false) : InstructionPredicateMatcher(IPM_Opcode, InsnVarID), - Insts(I.begin(), I.end()) { + Insts(I.begin(), I.end()), SkipCopies(SkipCopies) { assert((Insts.size() == 1 || Insts.size() == 2) && "unexpected number of opcode alternatives"); } @@ -1770,7 +1772,8 @@ bool isIdentical(const PredicateMatcher &B) const override { return InstructionPredicateMatcher::isIdentical(B) && - Insts == cast(&B)->Insts; + Insts == cast(&B)->Insts && + SkipCopies == cast(&B)->SkipCopies; } bool hasValue() const override { @@ -1792,8 +1795,10 @@ void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { - StringRef CheckType = Insts.size() == 1 ? - "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither"; + StringRef CheckType = SkipCopies + ? "GIM_CheckOpcodeIgnoreCopies" + : (Insts.size() == 1 ? "GIM_CheckOpcode" + : "GIM_CheckOpcodeIsEither"); Table << MatchTable::Opcode(CheckType) << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID); @@ -3891,6 +3896,11 @@ return failedImport( "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); } else { + bool SkipCopies = false; + if (Src->getOperator()->getName() == "g_maybe_cross_reg_bank_copy") { + Src = Src->getChild(0); + SkipCopies = true; + } SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); if (!SrcGIEquivOrNull) return failedImport("Pattern operator lacks an equivalent Instruction" + @@ -3898,7 +3908,7 @@ SrcGIOrNull = getEquivNode(*SrcGIEquivOrNull, Src); // The operators look good: match the opcode - InsnMatcher.addPredicate(SrcGIOrNull); + InsnMatcher.addPredicate(SrcGIOrNull, SkipCopies); } unsigned OpIdx = 0;