Index: llvm/trunk/lib/Target/AMDGPU/AMDGPU.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPU.td +++ llvm/trunk/lib/Target/AMDGPU/AMDGPU.td @@ -7,6 +7,7 @@ // //===------------------------------------------------------------===// +include "llvm/TableGen/SearchableTable.td" include "llvm/Target/Target.td" //===------------------------------------------------------------===// Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUSearchableTables.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUSearchableTables.td +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUSearchableTables.td @@ -7,8 +7,6 @@ // //===----------------------------------------------------------------------===// -include "llvm/TableGen/SearchableTable.td" - //===----------------------------------------------------------------------===// // Resource intrinsics table. //===----------------------------------------------------------------------===// Index: llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ llvm/trunk/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -1098,14 +1098,7 @@ AMDGPUOperand::Ptr defaultGLC() const; AMDGPUOperand::Ptr defaultSLC() const; - AMDGPUOperand::Ptr defaultTFE() const; - AMDGPUOperand::Ptr defaultD16() const; - AMDGPUOperand::Ptr defaultDMask() const; - AMDGPUOperand::Ptr defaultUNorm() const; - AMDGPUOperand::Ptr defaultDA() const; - AMDGPUOperand::Ptr defaultR128() const; - AMDGPUOperand::Ptr defaultLWE() const; AMDGPUOperand::Ptr defaultSMRDOffset8() const; AMDGPUOperand::Ptr defaultSMRDOffset20() const; AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const; @@ -4111,10 +4104,6 @@ return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC); } -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultTFE() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyTFE); -} - void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic, @@ -4271,30 +4260,6 @@ cvtMIMG(Inst, Operands, true); } -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDMask() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDMask); -} - -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultUNorm() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyUNorm); -} - -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDA() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDA); -} - -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultR128() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyR128); -} - -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultLWE() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyLWE); -} - -AMDGPUOperand::Ptr AMDGPUAsmParser::defaultD16() const { - return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyD16); -} - //===----------------------------------------------------------------------===// // smrd //===----------------------------------------------------------------------===// Index: llvm/trunk/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ llvm/trunk/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -329,19 +329,15 @@ int NewOpcode = -1; - if (IsAtomic) { - if (DMask == 0x1 || DMask == 0x3 || DMask == 0xF) { - NewOpcode = AMDGPU::getMaskedMIMGAtomicOp(*MCII, MI.getOpcode(), DstSize); - } - if (NewOpcode == -1) return MCDisassembler::Success; - } else if (IsGather4) { + if (IsGather4) { if (D16 && AMDGPU::hasPackedD16(STI)) - NewOpcode = AMDGPU::getMIMGGatherOpPackedD16(MI.getOpcode()); + NewOpcode = AMDGPU::getMaskedMIMGOp(MI.getOpcode(), 2); else return MCDisassembler::Success; } else { - NewOpcode = AMDGPU::getMaskedMIMGOp(*MCII, MI.getOpcode(), DstSize); - assert(NewOpcode != -1 && "could not find matching mimg channel instruction"); + NewOpcode = AMDGPU::getMaskedMIMGOp(MI.getOpcode(), DstSize); + if (NewOpcode == -1) + return MCDisassembler::Success; } auto RCID = MCII->get(NewOpcode).OpInfo[VDataIdx].RegClass; Index: llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td +++ llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td @@ -7,19 +7,46 @@ // //===----------------------------------------------------------------------===// -class MIMG_Mask { - string Op = op; - int Channels = channels; -} - -class MIMG_Atomic_Size { - string Op = op; - int AtomicSize = !if(is32Bit, 1, 2); -} +// MIMG-specific encoding families to distinguish between semantically +// equivalent machine instructions with different encoding. +// +// - MIMGEncPseudo: pseudo instruction, only used for atomics +// - MIMGEncGfx6: encoding introduced with gfx6 (obsoleted for atomics in gfx8) +// - MIMGEncGfx8: encoding introduced with gfx8 for atomics +class MIMGEncoding; + +def MIMGEncPseudo : MIMGEncoding; +def MIMGEncGfx6 : MIMGEncoding; +def MIMGEncGfx8 : MIMGEncoding; + +def MIMGEncoding : GenericEnum { + let FilterClass = "MIMGEncoding"; +} + +// Represent an ISA-level opcode, independent of the encoding and the +// vdata/vaddr size. +class MIMGBaseOpcode { + MIMGBaseOpcode BaseOpcode = !cast(NAME); + bits<8> NumExtraArgs = 0; + bit Gradients = 0; + bit Coordinates = 1; + bit LodOrClampOrMip = 0; + bit HasD16 = 0; +} + +def MIMGBaseOpcode : GenericEnum { + let FilterClass = "MIMGBaseOpcode"; +} + +def MIMGBaseOpcodesTable : GenericTable { + let FilterClass = "MIMGBaseOpcode"; + let CppTypeName = "MIMGBaseOpcodeInfo"; + let Fields = ["BaseOpcode", "NumExtraArgs", "Gradients", "Coordinates", + "LodOrClampOrMip", "HasD16"]; + GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode; -class MIMG_Gather_Size { - string Op = op; - int Channels = channels; + let PrimaryKey = ["BaseOpcode"]; + let PrimaryKeyName = "getMIMGBaseOpcodeInfo"; } class mimg si, bits<7> vi = si> { @@ -27,114 +54,160 @@ field bits<7> VI = vi; } -class MIMG_Helper : MIMG { +class MIMG + : InstSI { + + let VM_CNT = 1; + let EXP_CNT = 1; + let MIMG = 1; + let Uses = [EXEC]; let mayLoad = 1; let mayStore = 0; let hasPostISelHook = 1; + let SchedRW = [WriteVMEM]; + let UseNamedOperandTable = 1; + let hasSideEffects = 0; // XXX ???? + + let SubtargetPredicate = isGCN; let DecoderNamespace = dns; let isAsmParserOnly = !if(!eq(dns,""), 1, 0); let AsmMatchConverter = "cvtMIMG"; let usesCustomInserter = 1; - let SchedRW = [WriteVMEM]; + + Instruction Opcode = !cast(NAME); + MIMGBaseOpcode BaseOpcode; + MIMGEncoding MIMGEncoding = MIMGEncGfx6; + bits<8> VDataDwords; + bits<8> VAddrDwords; +} + +def MIMGInfoTable : GenericTable { + let FilterClass = "MIMG"; + let CppTypeName = "MIMGInfo"; + let Fields = ["Opcode", "BaseOpcode", "MIMGEncoding", "VDataDwords", "VAddrDwords"]; + GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode; + GenericEnum TypeOf_MIMGEncoding = MIMGEncoding; + + let PrimaryKey = ["BaseOpcode", "MIMGEncoding", "VDataDwords", "VAddrDwords"]; + let PrimaryKeyName = "getMIMGOpcodeHelper"; +} + +def getMIMGInfo : SearchIndex { + let Table = MIMGInfoTable; + let Key = ["Opcode"]; } class MIMG_NoSampler_Helper op, string asm, RegisterClass dst_rc, RegisterClass addr_rc, - bit has_d16, string dns=""> - : MIMG_Helper <(outs dst_rc:$vdata), - !con((ins addr_rc:$vaddr, SReg_256:$srsrc, - DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, - R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), - !if(has_d16, (ins D16:$d16), (ins))), - asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" - #!if(has_d16, "$d16", ""), - dns>, + : MIMG <(outs dst_rc:$vdata), dns>, MIMGe { let ssamp = 0; + let d16 = !if(BaseOpcode.HasD16, ?, 0); - let HasD16 = has_d16; - let d16 = !if(HasD16, ?, 0); + let InOperandList = !con((ins addr_rc:$vaddr, SReg_256:$srsrc, + DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, + R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), + !if(BaseOpcode.HasD16, (ins D16:$d16), (ins))); + let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" + #!if(BaseOpcode.HasD16, "$d16", ""); } multiclass MIMG_NoSampler_Src_Helper op, string asm, RegisterClass dst_rc, - int channels, bit has_d16> { - def NAME # _V1 : MIMG_NoSampler_Helper , - MIMG_Mask; - def NAME # _V2 : MIMG_NoSampler_Helper , - MIMG_Mask; - def NAME # _V3 : MIMG_NoSampler_Helper , - MIMG_Mask; - def NAME # _V4 : MIMG_NoSampler_Helper , - MIMG_Mask; -} - -multiclass MIMG_NoSampler op, string asm, bit has_d16> { - defm _V1 : MIMG_NoSampler_Src_Helper ; - defm _V2 : MIMG_NoSampler_Src_Helper ; - defm _V3 : MIMG_NoSampler_Src_Helper ; - defm _V4 : MIMG_NoSampler_Src_Helper ; + bit enableDisasm> { + let VAddrDwords = 1 in + def NAME # _V1 : MIMG_NoSampler_Helper ; + let VAddrDwords = 2 in + def NAME # _V2 : MIMG_NoSampler_Helper ; + let VAddrDwords = 3 in + def NAME # _V3 : MIMG_NoSampler_Helper ; + let VAddrDwords = 4 in + def NAME # _V4 : MIMG_NoSampler_Helper ; +} + +multiclass MIMG_NoSampler op, string asm, bit has_d16, bit mip = 0, + bit isResInfo = 0> { + def "" : MIMGBaseOpcode { + let Coordinates = !if(isResInfo, 0, 1); + let LodOrClampOrMip = mip; + let HasD16 = has_d16; + } + + let BaseOpcode = !cast(NAME), + mayLoad = !if(isResInfo, 0, 1) in { + let VDataDwords = 1 in + defm _V1 : MIMG_NoSampler_Src_Helper ; + let VDataDwords = 2 in + defm _V2 : MIMG_NoSampler_Src_Helper ; + let VDataDwords = 3 in + defm _V3 : MIMG_NoSampler_Src_Helper ; + let VDataDwords = 4 in + defm _V4 : MIMG_NoSampler_Src_Helper ; + } } class MIMG_Store_Helper op, string asm, RegisterClass data_rc, RegisterClass addr_rc, - bit has_d16, string dns = ""> - : MIMG_Helper <(outs), - !con((ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, - DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, - R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), - !if(has_d16, (ins D16:$d16), (ins))), - asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" - #!if(has_d16, "$d16", ""), - dns>, + : MIMG <(outs), dns>, MIMGe { let ssamp = 0; + let d16 = !if(BaseOpcode.HasD16, ?, 0); + let mayLoad = 0; let mayStore = 1; let hasSideEffects = 0; let hasPostISelHook = 0; let DisableWQM = 1; - let HasD16 = has_d16; - let d16 = !if(HasD16, ?, 0); + let InOperandList = !con((ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, + DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, + R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), + !if(BaseOpcode.HasD16, (ins D16:$d16), (ins))); + let AsmString = asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" + #!if(BaseOpcode.HasD16, "$d16", ""); } multiclass MIMG_Store_Addr_Helper op, string asm, RegisterClass data_rc, - int channels, bit has_d16> { - def NAME # _V1 : MIMG_Store_Helper , - MIMG_Mask; - def NAME # _V2 : MIMG_Store_Helper , - MIMG_Mask; - def NAME # _V3 : MIMG_Store_Helper , - MIMG_Mask; - def NAME # _V4 : MIMG_Store_Helper , - MIMG_Mask; -} - -multiclass MIMG_Store op, string asm, bit has_d16> { - defm _V1 : MIMG_Store_Addr_Helper ; - defm _V2 : MIMG_Store_Addr_Helper ; - defm _V3 : MIMG_Store_Addr_Helper ; - defm _V4 : MIMG_Store_Addr_Helper ; + bit enableDisasm> { + let VAddrDwords = 1 in + def NAME # _V1 : MIMG_Store_Helper ; + let VAddrDwords = 2 in + def NAME # _V2 : MIMG_Store_Helper ; + let VAddrDwords = 3 in + def NAME # _V3 : MIMG_Store_Helper ; + let VAddrDwords = 4 in + def NAME # _V4 : MIMG_Store_Helper ; +} + +multiclass MIMG_Store op, string asm, bit has_d16, bit mip = 0> { + def "" : MIMGBaseOpcode { + let LodOrClampOrMip = mip; + let HasD16 = has_d16; + } + + let BaseOpcode = !cast(NAME) in { + let VDataDwords = 1 in + defm _V1 : MIMG_Store_Addr_Helper ; + let VDataDwords = 2 in + defm _V2 : MIMG_Store_Addr_Helper ; + let VDataDwords = 3 in + defm _V3 : MIMG_Store_Addr_Helper ; + let VDataDwords = 4 in + defm _V4 : MIMG_Store_Addr_Helper ; + } } class MIMG_Atomic_Helper : MIMG_Helper < - (outs data_rc:$vdst), - (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, - DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, - R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), - asm#" $vdst, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da", - !if(enableDasm, dns, "")> { + bit enableDasm = 0> + : MIMG <(outs data_rc:$vdst), !if(enableDasm, dns, "")> { let mayLoad = 1; let mayStore = 1; let hasSideEffects = 1; // FIXME: Remove this @@ -142,181 +215,141 @@ let DisableWQM = 1; let Constraints = "$vdst = $vdata"; let AsmMatchConverter = "cvtMIMGAtomic"; + + let InOperandList = (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, + DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, + R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da); + let AsmString = asm#" $vdst, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da"; } -class MIMG_Atomic_Real_si - : MIMG_Atomic_Helper, - SIMCInstr, - MIMGe { - let isCodeGenOnly = 0; - let AssemblerPredicates = [isSICI]; - let DisableDecoder = DisableSIDecoder; - let d16 = 0; -} - -class MIMG_Atomic_Real_vi - : MIMG_Atomic_Helper, - SIMCInstr, - MIMGe { - let isCodeGenOnly = 0; - let AssemblerPredicates = [isVI]; - let DisableDecoder = DisableVIDecoder; - let d16 = 0; -} - -multiclass MIMG_Atomic_Helper_m { - let isPseudo = 1, isCodeGenOnly = 1 in { +multiclass MIMG_Atomic_Helper_m { + let isPseudo = 1, isCodeGenOnly = 1, MIMGEncoding = MIMGEncPseudo in { def "" : MIMG_Atomic_Helper, - SIMCInstr; + SIMCInstr; } - let ssamp = 0 in { - def _si : MIMG_Atomic_Real_si, - MIMG_Atomic_Size; - - def _vi : MIMG_Atomic_Real_vi, - MIMG_Atomic_Size; + let ssamp = 0, d16 = 0, isCodeGenOnly = 0 in { + def _si : MIMG_Atomic_Helper, + SIMCInstr, + MIMGe { + let AssemblerPredicates = [isSICI]; + let DisableDecoder = DisableSIDecoder; + } + + def _vi : MIMG_Atomic_Helper, + SIMCInstr, + MIMGe { + let AssemblerPredicates = [isVI]; + let DisableDecoder = DisableVIDecoder; + let MIMGEncoding = MIMGEncGfx8; + } } } -multiclass MIMG_Atomic_Addr_Helper_m { // _V* variants have different address size, but the size is not encoded. // So only one variant can be disassembled. V1 looks the safest to decode. - defm _V1 : MIMG_Atomic_Helper_m ; - defm _V2 : MIMG_Atomic_Helper_m ; - defm _V3 : MIMG_Atomic_Helper_m ; - defm _V4 : MIMG_Atomic_Helper_m ; -} - -multiclass MIMG_Atomic { // 64-bit atomics - // _V* variants have different dst size, but the size is encoded implicitly, - // using dmask and tfe. Only 32-bit variant is registered with disassembler. - // Other variants are reconstructed by disassembler using dmask and tfe. - defm _V1 : MIMG_Atomic_Addr_Helper_m ; - defm _V2 : MIMG_Atomic_Addr_Helper_m ; -} - -class MIMG_Sampler_Helper op, string asm, - RegisterClass dst_rc, - RegisterClass src_rc, - bit wqm, bit has_d16, - string dns=""> - : MIMG_Helper <(outs dst_rc:$vdata), - !con((ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, - DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, - R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), - !if(has_d16, (ins D16:$d16), (ins))), - asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da" - #!if(has_d16, "$d16", ""), - dns>, - MIMGe { - let WQM = wqm; - - let HasD16 = has_d16; - let d16 = !if(HasD16, ?, 0); + let VAddrDwords = 1 in + defm _V1 : MIMG_Atomic_Helper_m ; + let VAddrDwords = 2 in + defm _V2 : MIMG_Atomic_Helper_m ; + let VAddrDwords = 3 in + defm _V3 : MIMG_Atomic_Helper_m ; + let VAddrDwords = 4 in + defm _V4 : MIMG_Atomic_Helper_m ; +} + +multiclass MIMG_Atomic { // 64-bit atomics + def "" : MIMGBaseOpcode; + + let BaseOpcode = !cast(NAME) in { + // _V* variants have different dst size, but the size is encoded implicitly, + // using dmask and tfe. Only 32-bit variant is registered with disassembler. + // Other variants are reconstructed by disassembler using dmask and tfe. + let VDataDwords = !if(isCmpSwap, 2, 1) in + defm _V1 : MIMG_Atomic_Addr_Helper_m ; + let VDataDwords = !if(isCmpSwap, 4, 2) in + defm _V2 : MIMG_Atomic_Addr_Helper_m ; + } } -multiclass MIMG_Sampler_Src_Helper op, string asm, - RegisterClass dst_rc, - int channels, bit wqm, bit has_d16> { - def _V1 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V2 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V3 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V4 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V8 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V16 : MIMG_Sampler_Helper , - MIMG_Mask; +class MIMG_Sampler_Helper op, string asm, RegisterClass dst_rc, + RegisterClass src_rc, string dns=""> + : MIMG <(outs dst_rc:$vdata), dns>, + MIMGe { + let d16 = !if(BaseOpcode.HasD16, ?, 0); + + let InOperandList = !con((ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, + DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, + R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da), + !if(BaseOpcode.HasD16, (ins D16:$d16), (ins))); + let AsmString = asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da" + #!if(BaseOpcode.HasD16, "$d16", ""); +} + +multiclass MIMG_Sampler_Src_Helper op, string asm, RegisterClass dst_rc, + bit enableDisasm = 0> { + let VAddrDwords = 1 in + def _V1 : MIMG_Sampler_Helper ; + let VAddrDwords = 2 in + def _V2 : MIMG_Sampler_Helper ; + let VAddrDwords = 3 in + def _V3 : MIMG_Sampler_Helper ; + let VAddrDwords = 4 in + def _V4 : MIMG_Sampler_Helper ; + let VAddrDwords = 8 in + def _V8 : MIMG_Sampler_Helper ; + let VAddrDwords = 16 in + def _V16 : MIMG_Sampler_Helper ; +} + +class MIMG_Sampler_BaseOpcode + : MIMGBaseOpcode { + let NumExtraArgs = !size(sample.ExtraAddrArgs); + let Gradients = sample.Gradients; + let LodOrClampOrMip = !ne(sample.LodOrClamp, ""); } multiclass MIMG_Sampler op, AMDGPUSampleVariant sample, bit wqm = 0, - bit has_d16 = 1, + bit isGetLod = 0, string asm = "image_sample"#sample.LowerCaseMod> { - defm _V1 : MIMG_Sampler_Src_Helper; - defm _V2 : MIMG_Sampler_Src_Helper; - defm _V3 : MIMG_Sampler_Src_Helper; - defm _V4 : MIMG_Sampler_Src_Helper; -} - -multiclass MIMG_Sampler_WQM op, AMDGPUSampleVariant sample> : MIMG_Sampler; - -class MIMG_Gather_Helper op, string asm, - RegisterClass dst_rc, - RegisterClass src_rc, - bit wqm, - string dns=""> - : MIMG <(outs dst_rc:$vdata), - (ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, - DMask:$dmask, UNorm:$unorm, GLC:$glc, SLC:$slc, - R128:$r128, TFE:$tfe, LWE:$lwe, DA:$da, D16:$d16), - asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da$d16", - []>, - MIMGe { - let mayLoad = 1; - let mayStore = 0; - - // DMASK was repurposed for GATHER4. 4 components are always - // returned and DMASK works like a swizzle - it selects - // the component to fetch. The only useful DMASK values are - // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns - // (red,red,red,red) etc.) The ISA document doesn't mention - // this. - // Therefore, disable all code which updates DMASK by setting this: - let Gather4 = 1; - let hasPostISelHook = 0; - let WQM = wqm; - let HasD16 = 1; + def "" : MIMG_Sampler_BaseOpcode { + let HasD16 = !if(isGetLod, 0, 1); + } - let DecoderNamespace = dns; - let isAsmParserOnly = !if(!eq(dns,""), 1, 0); + let BaseOpcode = !cast(NAME), WQM = wqm, + mayLoad = !if(isGetLod, 0, 1) in { + let VDataDwords = 1 in + defm _V1 : MIMG_Sampler_Src_Helper; + let VDataDwords = 2 in + defm _V2 : MIMG_Sampler_Src_Helper; + let VDataDwords = 3 in + defm _V3 : MIMG_Sampler_Src_Helper; + let VDataDwords = 4 in + defm _V4 : MIMG_Sampler_Src_Helper; + } } - -multiclass MIMG_Gather_Src_Helper op, string asm, - RegisterClass dst_rc, - int channels, bit wqm> { - def _V1 : MIMG_Gather_Helper , - MIMG_Gather_Size; - def _V2 : MIMG_Gather_Helper , - MIMG_Gather_Size; - def _V3 : MIMG_Gather_Helper , - MIMG_Gather_Size; - def _V4 : MIMG_Gather_Helper , - MIMG_Gather_Size; - def _V8 : MIMG_Gather_Helper , - MIMG_Gather_Size; - def _V16 : MIMG_Gather_Helper , - MIMG_Gather_Size; -} +multiclass MIMG_Sampler_WQM op, AMDGPUSampleVariant sample> + : MIMG_Sampler; multiclass MIMG_Gather op, AMDGPUSampleVariant sample, bit wqm = 0, string asm = "image_gather4"#sample.LowerCaseMod> { - defm _V2 : MIMG_Gather_Src_Helper; /* for packed D16 only */ - defm _V4 : MIMG_Gather_Src_Helper; + def "" : MIMG_Sampler_BaseOpcode { + let HasD16 = 1; + } + + let BaseOpcode = !cast(NAME), WQM = wqm, + Gather4 = 1, hasPostISelHook = 0 in { + let VDataDwords = 2 in + defm _V2 : MIMG_Sampler_Src_Helper; /* for packed D16 only */ + let VDataDwords = 4 in + defm _V4 : MIMG_Sampler_Src_Helper; + } } multiclass MIMG_Gather_WQM op, AMDGPUSampleVariant sample> @@ -325,24 +358,21 @@ //===----------------------------------------------------------------------===// // MIMG Instructions //===----------------------------------------------------------------------===// -let SubtargetPredicate = isGCN in { defm IMAGE_LOAD : MIMG_NoSampler <0x00000000, "image_load", 1>; -defm IMAGE_LOAD_MIP : MIMG_NoSampler <0x00000001, "image_load_mip", 1>; +defm IMAGE_LOAD_MIP : MIMG_NoSampler <0x00000001, "image_load_mip", 1, 1>; defm IMAGE_LOAD_PCK : MIMG_NoSampler <0x00000002, "image_load_pck", 0>; defm IMAGE_LOAD_PCK_SGN : MIMG_NoSampler <0x00000003, "image_load_pck_sgn", 0>; -defm IMAGE_LOAD_MIP_PCK : MIMG_NoSampler <0x00000004, "image_load_mip_pck", 0>; -defm IMAGE_LOAD_MIP_PCK_SGN : MIMG_NoSampler <0x00000005, "image_load_mip_pck_sgn", 0>; +defm IMAGE_LOAD_MIP_PCK : MIMG_NoSampler <0x00000004, "image_load_mip_pck", 0, 1>; +defm IMAGE_LOAD_MIP_PCK_SGN : MIMG_NoSampler <0x00000005, "image_load_mip_pck_sgn", 0, 1>; defm IMAGE_STORE : MIMG_Store <0x00000008, "image_store", 1>; -defm IMAGE_STORE_MIP : MIMG_Store <0x00000009, "image_store_mip", 1>; +defm IMAGE_STORE_MIP : MIMG_Store <0x00000009, "image_store_mip", 1, 1>; defm IMAGE_STORE_PCK : MIMG_Store <0x0000000a, "image_store_pck", 0>; -defm IMAGE_STORE_MIP_PCK : MIMG_Store <0x0000000b, "image_store_mip_pck", 0>; +defm IMAGE_STORE_MIP_PCK : MIMG_Store <0x0000000b, "image_store_mip_pck", 0, 1>; -let mayLoad = 0, mayStore = 0 in { -defm IMAGE_GET_RESINFO : MIMG_NoSampler <0x0000000e, "image_get_resinfo", 0>; -} +defm IMAGE_GET_RESINFO : MIMG_NoSampler <0x0000000e, "image_get_resinfo", 0, 1, 1>; defm IMAGE_ATOMIC_SWAP : MIMG_Atomic , "image_atomic_swap">; -defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic , "image_atomic_cmpswap", VReg_64, VReg_128>; +defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic , "image_atomic_cmpswap", 1>; defm IMAGE_ATOMIC_ADD : MIMG_Atomic , "image_atomic_add">; defm IMAGE_ATOMIC_SUB : MIMG_Atomic , "image_atomic_sub">; //def IMAGE_ATOMIC_RSUB : MIMG_NoPattern_ <"image_atomic_rsub", 0x00000013>; -- not on VI @@ -355,7 +385,7 @@ defm IMAGE_ATOMIC_XOR : MIMG_Atomic , "image_atomic_xor">; defm IMAGE_ATOMIC_INC : MIMG_Atomic , "image_atomic_inc">; defm IMAGE_ATOMIC_DEC : MIMG_Atomic , "image_atomic_dec">; -//def IMAGE_ATOMIC_FCMPSWAP : MIMG_NoPattern_ <"image_atomic_fcmpswap", 0x0000001d>; -- not on VI +//def IMAGE_ATOMIC_FCMPSWAP : MIMG_NoPattern_ <"image_atomic_fcmpswap", 0x0000001d, 1>; -- not on VI //def IMAGE_ATOMIC_FMIN : MIMG_NoPattern_ <"image_atomic_fmin", 0x0000001e>; -- not on VI //def IMAGE_ATOMIC_FMAX : MIMG_NoPattern_ <"image_atomic_fmax", 0x0000001f>; -- not on VI defm IMAGE_SAMPLE : MIMG_Sampler_WQM <0x00000020, AMDGPUSample>; @@ -415,9 +445,7 @@ defm IMAGE_GATHER4_C_B_CL_O : MIMG_Gather_WQM <0x0000005e, AMDGPUSample_c_b_cl_o>; defm IMAGE_GATHER4_C_LZ_O : MIMG_Gather <0x0000005f, AMDGPUSample_c_lz_o>; -let mayLoad = 0, mayStore = 0 in { -defm IMAGE_GET_LOD : MIMG_Sampler <0x00000060, AMDGPUSample, 1, 0, "image_get_lod">; -} +defm IMAGE_GET_LOD : MIMG_Sampler <0x00000060, AMDGPUSample, 1, 1, "image_get_lod">; defm IMAGE_SAMPLE_CD : MIMG_Sampler <0x00000068, AMDGPUSample_cd>; defm IMAGE_SAMPLE_CD_CL : MIMG_Sampler <0x00000069, AMDGPUSample_cd_cl>; @@ -429,7 +457,6 @@ defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler <0x0000006f, AMDGPUSample_c_cd_cl_o>; //def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", 0x0000007e>; //def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", 0x0000007f>; -} /********** ============================== **********/ /********** Dimension-aware image patterns **********/ @@ -541,7 +568,7 @@ 0, /* tfe */ 0 /*(as_i1imm $lwe)*/, { I.P.Dim.DA }), - !if(MI.HasD16, (MI d16), (MI))); + !if(MI.BaseOpcode.HasD16, (MI d16), (MI))); let ResultInstrs = [ !if(IsCmpSwap, (EXTRACT_SUBREG ImageInstruction, sub0), ImageInstruction) ]; @@ -631,7 +658,7 @@ !con((opcode $addr, $rsrc, $sampler, (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), 0, 0, (as_i1imm $lwe), (as_i1imm $da)), - !if(opcode.HasD16, (opcode d16), (opcode))) + !if(opcode.BaseOpcode.HasD16, (opcode d16), (opcode))) >; } @@ -693,7 +720,7 @@ i1:$da)), !con((opcode $addr, $rsrc, (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), 0, 0, (as_i1imm $lwe), (as_i1imm $da)), - !if(opcode.HasD16, (opcode d16), (opcode))) + !if(opcode.BaseOpcode.HasD16, (opcode d16), (opcode))) >; } @@ -738,7 +765,7 @@ i1:$lwe, i1:$da), !con((opcode $data, $addr, $rsrc, (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), 0, 0, (as_i1imm $lwe), (as_i1imm $da)), - !if(opcode.HasD16, (opcode d16), (opcode))) + !if(opcode.BaseOpcode.HasD16, (opcode d16), (opcode))) >; } Index: llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp @@ -7821,9 +7821,7 @@ unsigned BitsSet = countPopulation(NewDmask); - const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); - int NewOpcode = AMDGPU::getMaskedMIMGOp(*TII, - Node->getMachineOpcode(), BitsSet); + int NewOpcode = AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), BitsSet); assert(NewOpcode != -1 && NewOpcode != static_cast(Node->getMachineOpcode()) && "failed to find equivalent MIMG op"); Index: llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td +++ llvm/trunk/lib/Target/AMDGPU/SIInstrFormats.td @@ -333,17 +333,3 @@ } } // End Uses = [EXEC] - -class MIMG pattern> : - InstSI { - - let VM_CNT = 1; - let EXP_CNT = 1; - let MIMG = 1; - let Uses = [EXEC]; - - let UseNamedOperandTable = 1; - let hasSideEffects = 0; // XXX ???? - - bit HasD16 = 0; -} Index: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td +++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td @@ -2021,62 +2021,6 @@ let ValueCols = [["Default"]]; } -def getMaskedMIMGOp1 : InstrMapping { - let FilterClass = "MIMG_Mask"; - let RowFields = ["Op"]; - let ColFields = ["Channels"]; - let KeyCol = ["1"]; - let ValueCols = [["2"], ["3"], ["4"] ]; -} - -def getMaskedMIMGOp2 : InstrMapping { - let FilterClass = "MIMG_Mask"; - let RowFields = ["Op"]; - let ColFields = ["Channels"]; - let KeyCol = ["2"]; - let ValueCols = [["1"], ["3"], ["4"] ]; -} - -def getMaskedMIMGOp3 : InstrMapping { - let FilterClass = "MIMG_Mask"; - let RowFields = ["Op"]; - let ColFields = ["Channels"]; - let KeyCol = ["3"]; - let ValueCols = [["1"], ["2"], ["4"] ]; -} - -def getMaskedMIMGOp4 : InstrMapping { - let FilterClass = "MIMG_Mask"; - let RowFields = ["Op"]; - let ColFields = ["Channels"]; - let KeyCol = ["4"]; - let ValueCols = [["1"], ["2"], ["3"] ]; -} - -def getMIMGAtomicOp1 : InstrMapping { - let FilterClass = "MIMG_Atomic_Size"; - let RowFields = ["Op"]; - let ColFields = ["AtomicSize"]; - let KeyCol = ["1"]; - let ValueCols = [["2"]]; -} - -def getMIMGAtomicOp2 : InstrMapping { - let FilterClass = "MIMG_Atomic_Size"; - let RowFields = ["Op"]; - let ColFields = ["AtomicSize"]; - let KeyCol = ["2"]; - let ValueCols = [["1"]]; -} - -def getMIMGGatherOpPackedD16 : InstrMapping { - let FilterClass = "MIMG_Gather_Size"; - let RowFields = ["Op"]; - let ColFields = ["Channels"]; - let KeyCol = ["4"]; - let ValueCols = [["2"]]; -} - // Maps an commuted opcode to its original version def getCommuteOrig : InstrMapping { let FilterClass = "Commutable_REV"; Index: llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h =================================================================== --- llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -37,6 +37,11 @@ class Triple; namespace AMDGPU { + +#define GET_MIMGBaseOpcode_DECL +#define GET_MIMGEncoding_DECL +#include "AMDGPUGenSearchableTables.inc" + namespace IsaInfo { enum { @@ -158,15 +163,7 @@ int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx); LLVM_READONLY -int getMaskedMIMGOp(const MCInstrInfo &MII, - unsigned Opc, unsigned NewChannels); - -LLVM_READONLY -int getMaskedMIMGAtomicOp(const MCInstrInfo &MII, - unsigned Opc, unsigned NewChannels); - -LLVM_READONLY -int getMIMGGatherOpPackedD16(uint16_t Opcode); +int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels); LLVM_READONLY int getMCOpcode(uint16_t Opcode, unsigned Gen); Index: llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ llvm/trunk/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -99,79 +99,23 @@ namespace AMDGPU { -LLVM_READNONE -static inline Channels indexToChannel(unsigned Channel) { - switch (Channel) { - case 1: - return AMDGPU::Channels_1; - case 2: - return AMDGPU::Channels_2; - case 3: - return AMDGPU::Channels_3; - case 4: - return AMDGPU::Channels_4; - default: - llvm_unreachable("invalid MIMG channel"); - } -} - - -// FIXME: Need to handle d16 images correctly. -static unsigned rcToChannels(unsigned RCID) { - switch (RCID) { - case AMDGPU::VGPR_32RegClassID: - return 1; - case AMDGPU::VReg_64RegClassID: - return 2; - case AMDGPU::VReg_96RegClassID: - return 3; - case AMDGPU::VReg_128RegClassID: - return 4; - default: - llvm_unreachable("invalid MIMG register class"); - } -} +struct MIMGInfo { + uint16_t Opcode; + uint16_t BaseOpcode; + uint8_t MIMGEncoding; + uint8_t VDataDwords; + uint8_t VAddrDwords; +}; -int getMaskedMIMGOp(const MCInstrInfo &MII, unsigned Opc, unsigned NewChannels) { - AMDGPU::Channels Channel = AMDGPU::indexToChannel(NewChannels); - unsigned OrigChannels = rcToChannels(MII.get(Opc).OpInfo[0].RegClass); - if (NewChannels == OrigChannels) - return Opc; - - switch (OrigChannels) { - case 1: - return AMDGPU::getMaskedMIMGOp1(Opc, Channel); - case 2: - return AMDGPU::getMaskedMIMGOp2(Opc, Channel); - case 3: - return AMDGPU::getMaskedMIMGOp3(Opc, Channel); - case 4: - return AMDGPU::getMaskedMIMGOp4(Opc, Channel); - default: - llvm_unreachable("invalid MIMG channel"); - } -} +#define GET_MIMGInfoTable_IMPL +#include "AMDGPUGenSearchableTables.inc" -int getMaskedMIMGAtomicOp(const MCInstrInfo &MII, unsigned Opc, unsigned NewChannels) { - assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst) != -1); - assert(NewChannels == 1 || NewChannels == 2 || NewChannels == 4); - - unsigned OrigChannels = rcToChannels(MII.get(Opc).OpInfo[0].RegClass); - assert(OrigChannels == 1 || OrigChannels == 2 || OrigChannels == 4); - - if (NewChannels == OrigChannels) return Opc; - - if (OrigChannels <= 2 && NewChannels <= 2) { - // This is an ordinary atomic (not an atomic_cmpswap) - return (OrigChannels == 1)? - AMDGPU::getMIMGAtomicOp1(Opc) : AMDGPU::getMIMGAtomicOp2(Opc); - } else if (OrigChannels >= 2 && NewChannels >= 2) { - // This is an atomic_cmpswap - return (OrigChannels == 2)? - AMDGPU::getMIMGAtomicOp1(Opc) : AMDGPU::getMIMGAtomicOp2(Opc); - } else { // invalid OrigChannels/NewChannels value - return -1; - } +int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) { + const MIMGInfo *OrigInfo = getMIMGInfo(Opc); + const MIMGInfo *NewInfo = + getMIMGOpcodeHelper(OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, + NewChannels, OrigInfo->VAddrDwords); + return NewInfo ? NewInfo->Opcode : -1; } // Wrapper for Tablegen'd function. enum Subtarget is not defined in any