diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -254,6 +254,15 @@ /// - OtherOpIdx - Other operand index GIM_CheckIsSameOperand, + /// Predicates with 'let PredicateCodeUsesOperands = 1' need to examine some + /// named operands that will be recorded in RecordedOperands. Names of these + /// operands are referenced in predicate argument list. Emitter determines + /// StoreIdx(corresponds to the order in which names appear in argument list). + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - StoreIdx - Store location in RecordedOperands. + GIM_RecordNamedOperand, + /// Fail the current try-block, or completely fail to match if there is no /// current try-block. GIM_Reject, @@ -446,6 +455,11 @@ std::vector Renderers; RecordedMIVector MIs; DenseMap TempRegisters; + /// Named operands that predicate with 'let PredicateCodeUsesOperands = 1' + /// referenced in its argument list. Operands are inserted at index set by + /// emitter, it corresponds to the order in which names appear in argument + /// list. Currently such predicates don't have more then 3 arguments. + std::array RecordedOperands; MatcherState(unsigned MaxRenderers); }; @@ -506,7 +520,9 @@ llvm_unreachable( "Subclasses must override this with a tablegen-erated function"); } - virtual bool testMIPredicate_MI(unsigned, const MachineInstr &) const { + virtual bool testMIPredicate_MI( + unsigned, const MachineInstr &, + const std::array &Operands) const { llvm_unreachable( "Subclasses must override this with a tablegen-erated function"); } diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -367,7 +367,8 @@ assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); - if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) + if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID], + State.RecordedOperands)) if (handleReject() == RejectAndGiveUp) return false; break; @@ -617,6 +618,20 @@ break; } + case GIM_RecordNamedOperand: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + uint64_t StoreIdx = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs[" + << InsnID << "]->getOperand(" << OpIdx + << "), StoreIdx=" << StoreIdx << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(StoreIdx < State.RecordedOperands.size() && "Index out of range"); + State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx); + break; + } case GIM_CheckRegBankForClass: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -50,6 +50,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector { private: MachineRegisterInfo *MRI; + const GCNSubtarget *Subtarget; public: AMDGPUInstructionSelector(const GCNSubtarget &STI, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -72,6 +72,7 @@ void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits &KB, CodeGenCoverage &CoverageInfo) { MRI = &MF.getRegInfo(); + Subtarget = &MF.getSubtarget(); InstructionSelector::setupMF(MF, KB, CoverageInfo); } diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -605,16 +605,24 @@ let PredicateCodeUsesOperands = 1; // The divergence predicate is irrelevant in GlobalISel, as we have - // proper register bank checks. We also force all VOP instruction - // operands to VGPR, so we should not need to check the constant bus - // restriction. + // proper register bank checks. We just need to verify the constant + // bus restriction when all the sources are considered. // // FIXME: With unlucky SGPR operands, we could penalize code by // blocking folding SGPR->VGPR copies later. // FIXME: There's no register bank verifier - // FIXME: Should add a way for the emitter to recognize this is a - // trivially true predicate to eliminate the check. - let GISelPredicateCode = [{return true;}]; + let GISelPredicateCode = [{ + const int ConstantBusLimit = Subtarget->getConstantBusLimit(AMDGPU::V_ADD3_U32); + int ConstantBusUses = 0; + for (unsigned i = 0; i < 3; ++i) { + const RegisterBank *RegBank = RBI.getRegBank(Operands[i]->getReg(), MRI, TRI); + if (RegBank->getID() == AMDGPU::SGPRRegBankID) { + if (++ConstantBusUses > ConstantBusLimit) + return false; + } + } + return true; + }]; } let SubtargetPredicate = isGFX9Plus in { diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll @@ -0,0 +1,149 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s +; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s + +; =================================================================================== +; V_ADD_LSHL_U32 +; =================================================================================== + +define amdgpu_ps float @add_shl(i32 %a, i32 %b, i32 %c) { +; VI-LABEL: add_shl: +; VI: ; %bb.0: +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_lshlrev_b32_e32 v0, v2, v0 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_add_lshl_u32 v0, v0, v1, v2 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_add_lshl_u32 v0, v0, v1, v2 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, %b + %result = shl i32 %x, %c + %bc = bitcast i32 %result to float + ret float %bc +} + +define amdgpu_ps float @add_shl_vgpr_c(i32 inreg %a, i32 inreg %b, i32 %c) { +; VI-LABEL: add_shl_vgpr_c: +; VI: ; %bb.0: +; VI-NEXT: s_add_i32 s2, s2, s3 +; VI-NEXT: v_lshlrev_b32_e64 v0, v0, s2 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl_vgpr_c: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_add_i32 s2, s2, s3 +; GFX9-NEXT: v_lshlrev_b32_e64 v0, v0, s2 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl_vgpr_c: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_add_i32 s2, s2, s3 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s2 +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, %b + %result = shl i32 %x, %c + %bc = bitcast i32 %result to float + ret float %bc +} + +define amdgpu_ps float @add_shl_vgpr_ac(i32 %a, i32 inreg %b, i32 %c) { +; VI-LABEL: add_shl_vgpr_ac: +; VI: ; %bb.0: +; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0 +; VI-NEXT: v_lshlrev_b32_e32 v0, v1, v0 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl_vgpr_ac: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_add_lshl_u32 v0, v0, s2, v1 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl_vgpr_ac: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_add_lshl_u32 v0, v0, s2, v1 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, %b + %result = shl i32 %x, %c + %bc = bitcast i32 %result to float + ret float %bc +} + +define amdgpu_ps float @add_shl_vgpr_const(i32 %a, i32 %b) { +; VI-LABEL: add_shl_vgpr_const: +; VI: ; %bb.0: +; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1 +; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl_vgpr_const: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_add_lshl_u32 v0, v0, v1, 9 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl_vgpr_const: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_add_lshl_u32 v0, v0, v1, 9 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, %b + %result = shl i32 %x, 9 + %bc = bitcast i32 %result to float + ret float %bc +} + +define amdgpu_ps float @add_shl_vgpr_const_inline_const(i32 %a) { +; VI-LABEL: add_shl_vgpr_const_inline_const: +; VI: ; %bb.0: +; VI-NEXT: v_add_u32_e32 v0, vcc, 0x3f4, v0 +; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl_vgpr_const_inline_const: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mov_b32_e32 v1, 0x3f4 +; GFX9-NEXT: v_add_lshl_u32 v0, v0, v1, 9 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl_vgpr_const_inline_const: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_add_lshl_u32 v0, v0, 0x3f4, 9 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, 1012 + %result = shl i32 %x, 9 + %bc = bitcast i32 %result to float + ret float %bc +} + +define amdgpu_ps float @add_shl_vgpr_inline_const_x2(i32 %a) { +; VI-LABEL: add_shl_vgpr_inline_const_x2: +; VI: ; %bb.0: +; VI-NEXT: v_add_u32_e32 v0, vcc, 3, v0 +; VI-NEXT: v_lshlrev_b32_e32 v0, 9, v0 +; VI-NEXT: ; return to shader part epilog +; +; GFX9-LABEL: add_shl_vgpr_inline_const_x2: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_add_lshl_u32 v0, v0, 3, 9 +; GFX9-NEXT: ; return to shader part epilog +; +; GFX10-LABEL: add_shl_vgpr_inline_const_x2: +; GFX10: ; %bb.0: +; GFX10-NEXT: v_add_lshl_u32 v0, v0, 3, 9 +; GFX10-NEXT: ; implicit-def: $vcc_hi +; GFX10-NEXT: ; return to shader part epilog + %x = add i32 %a, 3 + %result = shl i32 %x, 9 + %bc = bitcast i32 %result to float + ret float %bc +} diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -78,7 +78,7 @@ // CHECK-NEXT: bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) const override; // CHECK-NEXT: bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat &Imm) const override; // CHECK-NEXT: const int64_t *getMatchTable() const override; -// CHECK-NEXT: bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI) const override; +// CHECK-NEXT: bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const std::array &Operands) const override; // CHECK-NEXT: #endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL // CHECK-LABEL: #ifdef GET_GLOBALISEL_TEMPORARIES_INIT diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td --- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td +++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td @@ -45,61 +45,67 @@ let GISelPredicateCode = [{ return doesComplexCheck(MI); }]; + let PredicateCodeUsesOperands = 1; } -// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ {{[0-9]+}}, // Rule ID 1 // +// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 2 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, -// CHECK-NEXT: // MIs[0] Operand 1 +// CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:2:z +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] Operand 2 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1] // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR, // CHECK-NEXT: // MIs[1] Operand 0 -// CHECK-NEXT:GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[1] src0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[1] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, -// CHECK-NEXT: // MIs[0] src2 -// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1), DOP:{ *:[i32] }:$src2)<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:2:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y))<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, - -// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ {{[0-9]+}}, // Rule ID 2 // +// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 1 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, -// CHECK-NEXT: // MIs[0] src2 +// CHECK-NEXT: // MIs[0] Operand 1 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID, -// CHECK-NEXT: // MIs[0] Operand 2 -// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1] +// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1] // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR, // CHECK-NEXT: // MIs[1] Operand 0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[1] src0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:2:x // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[1] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:2:y // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID, +// CHECK-NEXT: // MIs[0] src2 +// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:2:z +// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_and_or_pat, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1))<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) +// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:2:x, DOP:{ *:[i32] }:$src1:$pred:2:y), DOP:{ *:[i32] }:$src2:$pred:2:z)<> => (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR, // Test commutative, standalone pattern. @@ -115,9 +121,11 @@ let GISelPredicateCode = [{ return doesComplexCheck(MI); }]; + + let PredicateCodeUsesOperands = 1; } -// CHECK: GIM_Try, /*On fail goto*//*Label 2*/ {{[0-9]+}}, // Rule ID 0 // +// CHECK: GIM_Try, /*On fail goto*//*Label 2*/ 285, // Rule ID 0 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, // CHECK-NEXT: // MIs[0] dst @@ -132,13 +140,16 @@ // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[1] src0 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:1:x // CHECK-NEXT: // MIs[1] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:1:y // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:1:z // CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIPFP_MI_Predicate_sub3_pat, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1), i32:{ *:[i32] }:$src2)<> => (SUB3:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) +// CHECK-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:1:x, i32:{ *:[i32] }:$src1:$pred:1:y), i32:{ *:[i32] }:$src2:$pred:1:z)<> => (SUB3:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUB3, // Test a non-commutative pattern. diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -389,6 +389,10 @@ return Name; } +static std::string getScopedName(unsigned Scope, const std::string &Name) { + return ("pred:" + Twine(Scope) + ":" + Name).str(); +} + //===- MatchTable Helpers -------------------------------------------------===// class MatchTable; @@ -1102,6 +1106,7 @@ OPM_PointerToAny, OPM_RegBank, OPM_MBB, + OPM_RecordNamedOperand, }; protected: @@ -1290,6 +1295,40 @@ } }; +/// Generates code to record named operand in RecordedOperands list at StoreIdx. +/// Predicates with 'let PredicateCodeUsesOperands = 1' get RecordedOperands as +/// an argument to predicate's c++ code once all operands have been matched. +class RecordNamedOperandMatcher : public OperandPredicateMatcher { +protected: + unsigned StoreIdx; + std::string Name; + +public: + RecordNamedOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + unsigned StoreIdx, StringRef Name) + : OperandPredicateMatcher(OPM_RecordNamedOperand, InsnVarID, OpIdx), + StoreIdx(StoreIdx), Name(Name) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_RecordNamedOperand; + } + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + StoreIdx == cast(&B)->StoreIdx && + Name.compare(cast(&B)->Name) == 0; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_RecordNamedOperand") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("StoreIdx") << MatchTable::IntValue(StoreIdx) + << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak; + } +}; + /// Generates code to check that an operand is a particular target constant. class ComplexPatternOperandMatcher : public OperandPredicateMatcher { protected: @@ -3459,6 +3498,16 @@ // Rule coverage information. Optional RuleCoverage; + /// Variables used to help with collecting of named operands for predicates + /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set + /// to the number of named operands that predicate expects. Store locations in + /// StoreIdxForName correspond to the order in which operand names appear in + /// predicate's argument list. + /// When we visit named leaf operand and WaitingForNamedOperands is not zero, + /// add matcher that will record operand and decrease counter. + unsigned WaitingForNamedOperands = 0; + StringMap StoreIdxForName; + void gatherOpcodeValues(); void gatherTypeIDValues(); void gatherNodeEquivs(); @@ -3511,7 +3560,8 @@ void emitCxxPredicateFns(raw_ostream &OS, StringRef CodeFieldName, StringRef TypeIdentifier, StringRef ArgType, - StringRef ArgName, StringRef AdditionalDeclarations, + StringRef ArgName, StringRef AdditionalArgs, + StringRef AdditionalDeclarations, std::function Filter); void emitImmPredicateFns(raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType, @@ -3863,6 +3913,15 @@ return std::move(Error); if (Predicate.hasGISelPredicateCode()) { + if (Predicate.usesOperands()) { + assert(WaitingForNamedOperands == 0 && + "previous predicate didn't find all operands or " + "nested predicate that uses operands"); + TreePattern *TP = Predicate.getOrigPatFragRecord(); + WaitingForNamedOperands = TP->getNumArgs(); + for (unsigned i = 0; i < WaitingForNamedOperands; ++i) + StoreIdxForName[getScopedName(Call.Scope, TP->getArgName(i))] = i; + } InsnMatcher.addPredicate(Predicate); continue; } @@ -4141,6 +4200,13 @@ if (auto *ChildDefInit = dyn_cast(SrcChild->getLeafValue())) { auto *ChildRec = ChildDefInit->getDef(); + if (WaitingForNamedOperands) { + auto PA = SrcChild->getNamesAsPredicateArg().begin(); + std::string Name = getScopedName(PA->getScope(), PA->getIdentifier()); + OM.addPredicate(StoreIdxForName[Name], Name); + --WaitingForNamedOperands; + } + // Check for register classes. if (ChildRec->isSubClassOf("RegisterClass") || ChildRec->isSubClassOf("RegisterOperand")) { @@ -5236,7 +5302,8 @@ // trouble than it's worth. void GlobalISelEmitter::emitCxxPredicateFns( raw_ostream &OS, StringRef CodeFieldName, StringRef TypeIdentifier, - StringRef ArgType, StringRef ArgName, StringRef AdditionalDeclarations, + StringRef ArgType, StringRef ArgName, StringRef AdditionalArgs, + StringRef AdditionalDeclarations, std::function Filter) { std::vector MatchedRecords; const auto &Defs = RK.getAllDerivedDefinitions("PatFrag"); @@ -5261,7 +5328,7 @@ OS << "bool " << Target.getName() << "InstructionSelector::test" << ArgName << "Predicate_" << TypeIdentifier << "(unsigned PredicateID, " << ArgType << " " - << ArgName << ") const {\n" + << ArgName << AdditionalArgs <<") const {\n" << AdditionalDeclarations; if (!AdditionalDeclarations.empty()) OS << "\n"; @@ -5287,12 +5354,13 @@ raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType, std::function Filter) { return emitCxxPredicateFns(OS, "ImmediateCode", TypeIdentifier, ArgType, - "Imm", "", Filter); + "Imm", "", "", Filter); } void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) { return emitCxxPredicateFns( OS, "GISelPredicateCode", "MI", "const MachineInstr &", "MI", + ", const std::array &Operands", " const MachineFunction &MF = *MI.getParent()->getParent();\n" " const MachineRegisterInfo &MRI = MF.getRegInfo();\n" " (void)MRI;", @@ -5525,7 +5593,8 @@ << " bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat " "&Imm) const override;\n" << " const int64_t *getMatchTable() const override;\n" - << " bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI) " + << " bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI" + ", const std::array &Operands) " "const override;\n" << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n";