Index: llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -313,6 +313,14 @@ /// - RendererFnID - Custom renderer function to call GIR_CustomRenderer, + /// Render operands to the specified instruction using a custom function, + /// reading from a specific operand. + /// - InsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to get the matched operand from + /// - OpIdx - Operand index in OldInsnID the render function should read from.. + /// - RendererFnID - Custom renderer function to call + GIR_CustomOperandRenderer, + /// Render a G_CONSTANT operator as a sign-extended immediate. /// - NewInsnID - Instruction ID to modify /// - OldInsnID - Instruction ID to copy from Index: llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h =================================================================== --- llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -947,8 +947,27 @@ dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" << InsnID << "], MIs[" << OldInsnID << "], " << RendererFnID << ")\n"); + (ISel.*ISelInfo.CustomRenderers[RendererFnID])( + OutMIs[InsnID], *State.MIs[OldInsnID], + -1); // Not a source operand of the old instruction. + break; + } + case GIR_CustomOperandRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t RendererFnID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CustomOperandRenderer(OutMIs[" + << InsnID << "], MIs[" << OldInsnID << "]->getOperand(" + << OpIdx << "), " + << RendererFnID << ")\n"); (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], - *State.MIs[OldInsnID]); + *State.MIs[OldInsnID], + OpIdx); break; } case GIR_ConstrainOperandRC: { Index: llvm/include/llvm/Target/GlobalISel/Target.td =================================================================== --- llvm/include/llvm/Target/GlobalISel/Target.td +++ llvm/include/llvm/Target/GlobalISel/Target.td @@ -55,6 +55,12 @@ class GICustomOperandRenderer { // The function renders the operand(s) of the matched instruction to // the specified instruction. It should be of the form: - // void render(MachineInstrBuilder &MIB, const MachineInstr &MI) + // void render(MachineInstrBuilder &MIB, const MachineInstr &MI, + // int OpIdx = -1) + // + // If OpIdx is specified (i.e. not invalid/negative), this + // references the source operand MI.getOperand(OpIdx). Otherwise, + // this is the value defined by MI. This is to support the case + // where there is no corresponding instruction to match. string RendererFn = rendererfn; } Index: llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -236,9 +236,12 @@ MachineIRBuilder &MIB) const; ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const; - void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const; - void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I) const; - void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I) const; + void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx = -1) const; + void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I, + int OpIdx = -1) const; + void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I, + int OpIdx = -1) const; // Materialize a GlobalValue or BlockAddress using a movz+movk sequence. void materializeLargeCMVal(MachineInstr &I, const Value *V, @@ -4702,25 +4705,29 @@ } void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { + const MachineInstr &MI, + int OpIdx) const { const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); - assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); Optional CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI); assert(CstVal && "Expected constant value"); MIB.addImm(CstVal.getValue()); } void AArch64InstructionSelector::renderLogicalImm32( - MachineInstrBuilder &MIB, const MachineInstr &I) const { - assert(I.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { + assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue(); uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 32); MIB.addImm(Enc); } void AArch64InstructionSelector::renderLogicalImm64( - MachineInstrBuilder &MIB, const MachineInstr &I) const { - assert(I.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { + assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue(); uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64); MIB.addImm(Enc); Index: llvm/lib/Target/AMDGPU/AMDGPUGISel.td =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -209,6 +209,9 @@ def gi_as_i32timm : GICustomOperandRenderer<"renderTruncImm32">, GISDNodeXFormEquiv; +def gi_as_i16timm : GICustomOperandRenderer<"renderTruncTImm">, + GISDNodeXFormEquiv; + def gi_NegateImm : GICustomOperandRenderer<"renderNegateImm">, GISDNodeXFormEquiv; Index: llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -168,17 +168,20 @@ InstructionSelector::ComplexRendererFns selectDS1Addr1Offset(MachineOperand &Root) const; - void renderTruncImm32(MachineInstrBuilder &MIB, - const MachineInstr &MI) const; + void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx = -1) const; - void renderNegateImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const; + void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; - void renderBitcastImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const; + void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; - void renderPopcntImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const; + void renderBitcastImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; + + void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; bool isInlineImmediate16(int64_t Imm) const; bool isInlineImmediate32(int64_t Imm) const; Index: llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -2106,21 +2106,28 @@ } void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { - assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); Optional CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), *MRI); assert(CstVal && "Expected constant value"); MIB.addImm(CstVal.getValue()); } void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { - assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue()); } void AMDGPUInstructionSelector::renderBitcastImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { + const MachineInstr &MI, + int OpIdx) const { + assert(OpIdx == -1); + const MachineOperand &Op = MI.getOperand(1); if (MI.getOpcode() == TargetOpcode::G_FCONSTANT) MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); @@ -2131,11 +2138,21 @@ } void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { - assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); MIB.addImm(MI.getOperand(1).getCImm()->getValue().countPopulation()); } +/// This only really exists to satisfy DAG type checking machinery, so is a +/// no-op here. +void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + MIB.addImm(MI.getOperand(OpIdx).getImm()); +} + bool AMDGPUInstructionSelector::isInlineImmediate16(int64_t Imm) const { return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); } Index: llvm/lib/Target/AMDGPU/DSInstructions.td =================================================================== --- llvm/lib/Target/AMDGPU/DSInstructions.td +++ llvm/lib/Target/AMDGPU/DSInstructions.td @@ -619,7 +619,7 @@ def : GCNPat < (int_amdgcn_ds_swizzle i32:$src, timm:$offset16), - (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16imm $offset16), (i1 0)) + (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0)) >; class DSReadPat : GCNPat < Index: llvm/lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -677,6 +677,10 @@ return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16); }]>; +def as_i16timm : SDNodeXFormgetTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16); +}]>; + def as_i32imm: SDNodeXFormgetTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32); }]>; Index: llvm/lib/Target/ARM/ARMInstructionSelector.cpp =================================================================== --- llvm/lib/Target/ARM/ARMInstructionSelector.cpp +++ llvm/lib/Target/ARM/ARMInstructionSelector.cpp @@ -138,8 +138,10 @@ unsigned selectLoadStoreOpCode(unsigned Opc, unsigned RegBank, unsigned Size) const; - void renderVFPF32Imm(MachineInstrBuilder &New, const MachineInstr &Old) const; - void renderVFPF64Imm(MachineInstrBuilder &New, const MachineInstr &Old) const; + void renderVFPF32Imm(MachineInstrBuilder &New, const MachineInstr &Old, + int OpIdx = -1) const; + void renderVFPF64Imm(MachineInstrBuilder &New, const MachineInstr &Old, + int OpIdx = -1) const; #define GET_GLOBALISEL_PREDICATES_DECL #include "ARMGenGlobalISel.inc" @@ -811,9 +813,10 @@ } void ARMInstructionSelector::renderVFPF32Imm( - MachineInstrBuilder &NewInstBuilder, const MachineInstr &OldInst) const { + MachineInstrBuilder &NewInstBuilder, const MachineInstr &OldInst, + int OpIdx) const { assert(OldInst.getOpcode() == TargetOpcode::G_FCONSTANT && - "Expected G_FCONSTANT"); + OpIdx == -1 && "Expected G_FCONSTANT"); APFloat FPImmValue = OldInst.getOperand(1).getFPImm()->getValueAPF(); int FPImmEncoding = ARM_AM::getFP32Imm(FPImmValue); @@ -823,9 +826,9 @@ } void ARMInstructionSelector::renderVFPF64Imm( - MachineInstrBuilder &NewInstBuilder, const MachineInstr &OldInst) const { + MachineInstrBuilder &NewInstBuilder, const MachineInstr &OldInst, int OpIdx) const { assert(OldInst.getOpcode() == TargetOpcode::G_FCONSTANT && - "Expected G_FCONSTANT"); + OpIdx == -1 && "Expected G_FCONSTANT"); APFloat FPImmValue = OldInst.getOperand(1).getFPImm()->getValueAPF(); int FPImmEncoding = ARM_AM::getFP64Imm(FPImmValue); Index: llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ds.swizzle.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ds.swizzle.mir @@ -0,0 +1,46 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s + +--- + +name: ds_swizzle_0 +legalized: true +regBankSelected: true +tracksRegLiveness: true + + +body: | + bb.0: + liveins: $vgpr0 + ; CHECK-LABEL: name: ds_swizzle_0 + ; CHECK: liveins: $vgpr0 + ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; CHECK: [[DS_SWIZZLE_B32_:%[0-9]+]]:vgpr_32 = DS_SWIZZLE_B32 [[COPY]], 0, 0, implicit $exec + ; CHECK: S_ENDPGM 0, implicit [[DS_SWIZZLE_B32_]] + %0:vgpr(s32) = COPY $vgpr0 + %1:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ds.swizzle), %0, 0 + S_ENDPGM 0, implicit %1 + +... + +--- + +name: ds_swizzle_65535 +legalized: true +regBankSelected: true +tracksRegLiveness: true + + +body: | + bb.0: + liveins: $vgpr0 + ; CHECK-LABEL: name: ds_swizzle_65535 + ; CHECK: liveins: $vgpr0 + ; CHECK: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; CHECK: [[DS_SWIZZLE_B32_:%[0-9]+]]:vgpr_32 = DS_SWIZZLE_B32 [[COPY]], 65535, 0, implicit $exec + ; CHECK: S_ENDPGM 0, implicit [[DS_SWIZZLE_B32_]] + %0:vgpr(s32) = COPY $vgpr0 + %1:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ds.swizzle), %0, 65535 + S_ENDPGM 0, implicit %1 + +... Index: llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td =================================================================== --- /dev/null +++ llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td @@ -0,0 +1,37 @@ +// RUN: llvm-tblgen -gen-global-isel -warn-on-skipped-patterns -optimize-match-table=false -I %p/../../include -I %p/Common %s -o - | FileCheck -check-prefix=GISEL %s + +include "llvm/Target/Target.td" +include "GlobalISelEmitterCommon.td" + +def shiftl_1 : SDNodeXFormgetTargetConstant(N->getZExtValue() << 1, SDLoc(N), MVT::i32); +}]>; + +def gi_shiftl_1 : GICustomOperandRenderer<"renderShiftImml1">, + GISDNodeXFormEquiv; + + +def int_mytarget_sleep : Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]>; +def int_mytarget_foo : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [ImmArg<1>, IntrNoMem]>; + + +def SLEEP : I<(outs), (ins i32imm:$src0), []>; +def FOO : I<(outs GPR32:$dst), (ins GPR32:$src0, i32imm:$src1), []>; + +// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC, +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::mytarget_foo, +// GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/3, +// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, /*OperandRenderer*/GICR_renderShiftImml1, // src1 +def : Pat< + (int_mytarget_foo i32:$src0, (i32 timm:$src1)), + (FOO GPR32:$src0, (shiftl_1 $src1)) +>; + +// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS, +// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep, +// GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/1, +// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*OperandRenderer*/GICR_renderShiftImml1, // src0 +def : Pat< + (int_mytarget_sleep (i32 timm:$src0)), + (SLEEP (shiftl_1 $src0)) +>; Index: llvm/test/TableGen/GlobalISelEmitter.td =================================================================== --- llvm/test/TableGen/GlobalISelEmitter.td +++ llvm/test/TableGen/GlobalISelEmitter.td @@ -70,7 +70,7 @@ // CHECK-LABEL: #ifdef GET_GLOBALISEL_TEMPORARIES_DECL // CHECK-NEXT: mutable MatcherState State; // CHECK-NEXT: typedef ComplexRendererFns(MyTargetInstructionSelector::*ComplexMatcherMemFn)(MachineOperand &) const; -// CHECK-NEXT: typedef void(MyTargetInstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const MachineInstr&) const; +// CHECK-NEXT: typedef void(MyTargetInstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const MachineInstr&, int) const; // CHECK-NEXT: const ISelInfoTy ISelInfo; // CHECK-NEXT: static MyTargetInstructionSelector::ComplexMatcherMemFn ComplexPredicateFns[]; // CHECK-NEXT: static MyTargetInstructionSelector::CustomRendererFn CustomRenderers[]; Index: llvm/utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- llvm/utils/TableGen/GlobalISelEmitter.cpp +++ llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -2318,7 +2318,8 @@ OR_Register, OR_TempRegister, OR_ComplexPattern, - OR_Custom + OR_Custom, + OR_CustomOperand }; protected: @@ -2726,6 +2727,38 @@ } }; +class CustomOperandRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const Record &Renderer; + /// The name of the operand. + const std::string SymbolicName; + +public: + CustomOperandRenderer(unsigned InsnID, const Record &Renderer, + StringRef SymbolicName) + : OperandRenderer(OR_CustomOperand), InsnID(InsnID), Renderer(Renderer), + SymbolicName(SymbolicName) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CustomOperand; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName); + Table << MatchTable::Opcode("GIR_CustomOperandRenderer") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OpdMatcher.getInsnVarID()) + << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(OpdMatcher.getOpIdx()) + << MatchTable::Comment("OperandRenderer") + << MatchTable::NamedValue( + "GICR_" + Renderer.getValueAsString("RendererFn").str()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + /// An action taken when all Matcher predicates succeeded for a parent rule. /// /// Typical actions include: @@ -3939,12 +3972,22 @@ } if (!DstChild->isLeaf()) { - if (DstChild->getOperator()->isSubClassOf("SDNodeXForm")) { auto Child = DstChild->getChild(0); auto I = SDNodeXFormEquivs.find(DstChild->getOperator()); if (I != SDNodeXFormEquivs.end()) { - DstMIBuilder.addRenderer(*I->second, Child->getName()); + Record *XFormOpc = DstChild->getOperator()->getValueAsDef("Opcode"); + if (XFormOpc->getName() == "timm") { + // If this is a TargetConstant, there won't be a corresponding + // instruction to transform. Instead, this will refer directly to an + // operand in an instruction's operand list. + DstMIBuilder.addRenderer(*I->second, + Child->getName()); + } else { + DstMIBuilder.addRenderer(*I->second, + Child->getName()); + } + return InsertPt; } return failedImport("SDNodeXForm " + Child->getName() + @@ -5112,7 +5155,7 @@ << " typedef void(" << Target.getName() << "InstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const " - "MachineInstr&) " + "MachineInstr&, int) " "const;\n" << " const ISelInfoTy "