Index: llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td +++ llvm/trunk/lib/Target/AMDGPU/MIMGInstructions.td @@ -0,0 +1,750 @@ +//===-- MIMGInstructions.td - MIMG Instruction Defintions -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +class MIMG_Mask { + string Op = op; + int Channels = channels; +} + +class mimg si, bits<7> vi = si> { + field bits<7> SI = si; + field bits<7> VI = vi; +} + +class MIMG_Helper : MIMG { + let mayLoad = 1; + let mayStore = 0; + let hasPostISelHook = 1; + let DecoderNamespace = dns; + let isAsmParserOnly = !if(!eq(dns,""), 1, 0); + let AsmMatchConverter = "cvtMIMG"; +} + +class MIMG_NoSampler_Helper op, string asm, + RegisterClass dst_rc, + RegisterClass addr_rc, + string dns=""> : MIMG_Helper < + (outs dst_rc:$vdata), + (ins addr_rc:$vaddr, SReg_256:$srsrc, + dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, + r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), + asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da", + dns>, MIMGe { + let ssamp = 0; +} + +multiclass MIMG_NoSampler_Src_Helper op, string asm, + RegisterClass dst_rc, + int channels> { + def _V1 : MIMG_NoSampler_Helper , + MIMG_Mask; + def _V2 : MIMG_NoSampler_Helper , + MIMG_Mask; + def _V4 : MIMG_NoSampler_Helper , + MIMG_Mask; +} + +multiclass MIMG_NoSampler op, string asm> { + defm _V1 : MIMG_NoSampler_Src_Helper ; + defm _V2 : MIMG_NoSampler_Src_Helper ; + defm _V3 : MIMG_NoSampler_Src_Helper ; + defm _V4 : MIMG_NoSampler_Src_Helper ; +} + +class MIMG_Store_Helper op, string asm, + RegisterClass data_rc, + RegisterClass addr_rc> : MIMG_Helper < + (outs), + (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, + dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, + r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), + asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" + >, MIMGe { + let ssamp = 0; + let mayLoad = 1; // TableGen requires this for matching with the intrinsics + let mayStore = 1; + let hasSideEffects = 1; + let hasPostISelHook = 0; + let DisableWQM = 1; +} + +multiclass MIMG_Store_Addr_Helper op, string asm, + RegisterClass data_rc, + int channels> { + def _V1 : MIMG_Store_Helper , + MIMG_Mask; + def _V2 : MIMG_Store_Helper , + MIMG_Mask; + def _V4 : MIMG_Store_Helper , + MIMG_Mask; +} + +multiclass MIMG_Store op, string asm> { + defm _V1 : MIMG_Store_Addr_Helper ; + defm _V2 : MIMG_Store_Addr_Helper ; + defm _V3 : MIMG_Store_Addr_Helper ; + defm _V4 : MIMG_Store_Addr_Helper ; +} + +class MIMG_Atomic_Helper : MIMG_Helper < + (outs data_rc:$vdst), + (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, + dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, + r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), + asm#" $vdst, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" + > { + let mayStore = 1; + let hasSideEffects = 1; + let hasPostISelHook = 0; + let DisableWQM = 1; + let Constraints = "$vdst = $vdata"; + let AsmMatchConverter = "cvtMIMGAtomic"; +} + +class MIMG_Atomic_Real_si : + MIMG_Atomic_Helper, + SIMCInstr, + MIMGe { + let isCodeGenOnly = 0; + let AssemblerPredicates = [isSICI]; + let DecoderNamespace = "SICI"; + let DisableDecoder = DisableSIDecoder; +} + +class MIMG_Atomic_Real_vi : + MIMG_Atomic_Helper, + SIMCInstr, + MIMGe { + let isCodeGenOnly = 0; + let AssemblerPredicates = [isVI]; + let DecoderNamespace = "VI"; + let DisableDecoder = DisableVIDecoder; +} + +multiclass MIMG_Atomic_Helper_m { + let isPseudo = 1, isCodeGenOnly = 1 in { + def "" : MIMG_Atomic_Helper, + SIMCInstr; + } + + let ssamp = 0 in { + def _si : MIMG_Atomic_Real_si; + + def _vi : MIMG_Atomic_Real_vi; + } +} + +multiclass MIMG_Atomic { + defm _V1 : MIMG_Atomic_Helper_m ; + defm _V2 : MIMG_Atomic_Helper_m ; + defm _V4 : MIMG_Atomic_Helper_m ; +} + +class MIMG_Sampler_Helper op, string asm, + RegisterClass dst_rc, + RegisterClass src_rc, + int wqm, + string dns=""> : MIMG_Helper < + (outs dst_rc:$vdata), + (ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, + dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, + r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), + asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da", + dns>, MIMGe { + let WQM = wqm; +} + +multiclass MIMG_Sampler_Src_Helper op, string asm, + RegisterClass dst_rc, + int channels, int wqm> { + def _V1 : MIMG_Sampler_Helper , + MIMG_Mask; + def _V2 : MIMG_Sampler_Helper , + MIMG_Mask; + def _V4 : MIMG_Sampler_Helper , + MIMG_Mask; + def _V8 : MIMG_Sampler_Helper , + MIMG_Mask; + def _V16 : MIMG_Sampler_Helper , + MIMG_Mask; +} + +multiclass MIMG_Sampler op, string asm, int wqm=0> { + defm _V1 : MIMG_Sampler_Src_Helper; + defm _V2 : MIMG_Sampler_Src_Helper; + defm _V3 : MIMG_Sampler_Src_Helper; + defm _V4 : MIMG_Sampler_Src_Helper; +} + +multiclass MIMG_Sampler_WQM op, string asm> : MIMG_Sampler; + +class MIMG_Gather_Helper op, string asm, + RegisterClass dst_rc, + RegisterClass src_rc, int wqm> : MIMG < + (outs dst_rc:$vdata), + (ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, + dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, + r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), + asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da", + []>, MIMGe { + let mayLoad = 1; + let mayStore = 0; + + // DMASK was repurposed for GATHER4. 4 components are always + // returned and DMASK works like a swizzle - it selects + // the component to fetch. The only useful DMASK values are + // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns + // (red,red,red,red) etc.) The ISA document doesn't mention + // this. + // Therefore, disable all code which updates DMASK by setting this: + let Gather4 = 1; + let hasPostISelHook = 0; + let WQM = wqm; + + let isAsmParserOnly = 1; // TBD: fix it later +} + +multiclass MIMG_Gather_Src_Helper op, string asm, + RegisterClass dst_rc, + int channels, int wqm> { + def _V1 : MIMG_Gather_Helper , + MIMG_Mask; + def _V2 : MIMG_Gather_Helper , + MIMG_Mask; + def _V4 : MIMG_Gather_Helper , + MIMG_Mask; + def _V8 : MIMG_Gather_Helper , + MIMG_Mask; + def _V16 : MIMG_Gather_Helper , + MIMG_Mask; +} + +multiclass MIMG_Gather op, string asm, int wqm=0> { + defm _V1 : MIMG_Gather_Src_Helper; + defm _V2 : MIMG_Gather_Src_Helper; + defm _V3 : MIMG_Gather_Src_Helper; + defm _V4 : MIMG_Gather_Src_Helper; +} + +multiclass MIMG_Gather_WQM op, string asm> : MIMG_Gather; + +//===----------------------------------------------------------------------===// +// MIMG Instructions +//===----------------------------------------------------------------------===// +let SubtargetPredicate = isGCN in { +defm IMAGE_LOAD : MIMG_NoSampler <0x00000000, "image_load">; +defm IMAGE_LOAD_MIP : MIMG_NoSampler <0x00000001, "image_load_mip">; +//def IMAGE_LOAD_PCK : MIMG_NoPattern_ <"image_load_pck", 0x00000002>; +//def IMAGE_LOAD_PCK_SGN : MIMG_NoPattern_ <"image_load_pck_sgn", 0x00000003>; +//def IMAGE_LOAD_MIP_PCK : MIMG_NoPattern_ <"image_load_mip_pck", 0x00000004>; +//def IMAGE_LOAD_MIP_PCK_SGN : MIMG_NoPattern_ <"image_load_mip_pck_sgn", 0x00000005>; +defm IMAGE_STORE : MIMG_Store <0x00000008, "image_store">; +defm IMAGE_STORE_MIP : MIMG_Store <0x00000009, "image_store_mip">; +//def IMAGE_STORE_PCK : MIMG_NoPattern_ <"image_store_pck", 0x0000000a>; +//def IMAGE_STORE_MIP_PCK : MIMG_NoPattern_ <"image_store_mip_pck", 0x0000000b>; +defm IMAGE_GET_RESINFO : MIMG_NoSampler <0x0000000e, "image_get_resinfo">; +defm IMAGE_ATOMIC_SWAP : MIMG_Atomic , "image_atomic_swap">; +defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic , "image_atomic_cmpswap", VReg_64>; +defm IMAGE_ATOMIC_ADD : MIMG_Atomic , "image_atomic_add">; +defm IMAGE_ATOMIC_SUB : MIMG_Atomic , "image_atomic_sub">; +//def IMAGE_ATOMIC_RSUB : MIMG_NoPattern_ <"image_atomic_rsub", 0x00000013>; -- not on VI +defm IMAGE_ATOMIC_SMIN : MIMG_Atomic , "image_atomic_smin">; +defm IMAGE_ATOMIC_UMIN : MIMG_Atomic , "image_atomic_umin">; +defm IMAGE_ATOMIC_SMAX : MIMG_Atomic , "image_atomic_smax">; +defm IMAGE_ATOMIC_UMAX : MIMG_Atomic , "image_atomic_umax">; +defm IMAGE_ATOMIC_AND : MIMG_Atomic , "image_atomic_and">; +defm IMAGE_ATOMIC_OR : MIMG_Atomic , "image_atomic_or">; +defm IMAGE_ATOMIC_XOR : MIMG_Atomic , "image_atomic_xor">; +defm IMAGE_ATOMIC_INC : MIMG_Atomic , "image_atomic_inc">; +defm IMAGE_ATOMIC_DEC : MIMG_Atomic , "image_atomic_dec">; +//def IMAGE_ATOMIC_FCMPSWAP : MIMG_NoPattern_ <"image_atomic_fcmpswap", 0x0000001d>; -- not on VI +//def IMAGE_ATOMIC_FMIN : MIMG_NoPattern_ <"image_atomic_fmin", 0x0000001e>; -- not on VI +//def IMAGE_ATOMIC_FMAX : MIMG_NoPattern_ <"image_atomic_fmax", 0x0000001f>; -- not on VI +defm IMAGE_SAMPLE : MIMG_Sampler_WQM <0x00000020, "image_sample">; +defm IMAGE_SAMPLE_CL : MIMG_Sampler_WQM <0x00000021, "image_sample_cl">; +defm IMAGE_SAMPLE_D : MIMG_Sampler <0x00000022, "image_sample_d">; +defm IMAGE_SAMPLE_D_CL : MIMG_Sampler <0x00000023, "image_sample_d_cl">; +defm IMAGE_SAMPLE_L : MIMG_Sampler <0x00000024, "image_sample_l">; +defm IMAGE_SAMPLE_B : MIMG_Sampler_WQM <0x00000025, "image_sample_b">; +defm IMAGE_SAMPLE_B_CL : MIMG_Sampler_WQM <0x00000026, "image_sample_b_cl">; +defm IMAGE_SAMPLE_LZ : MIMG_Sampler <0x00000027, "image_sample_lz">; +defm IMAGE_SAMPLE_C : MIMG_Sampler_WQM <0x00000028, "image_sample_c">; +defm IMAGE_SAMPLE_C_CL : MIMG_Sampler_WQM <0x00000029, "image_sample_c_cl">; +defm IMAGE_SAMPLE_C_D : MIMG_Sampler <0x0000002a, "image_sample_c_d">; +defm IMAGE_SAMPLE_C_D_CL : MIMG_Sampler <0x0000002b, "image_sample_c_d_cl">; +defm IMAGE_SAMPLE_C_L : MIMG_Sampler <0x0000002c, "image_sample_c_l">; +defm IMAGE_SAMPLE_C_B : MIMG_Sampler_WQM <0x0000002d, "image_sample_c_b">; +defm IMAGE_SAMPLE_C_B_CL : MIMG_Sampler_WQM <0x0000002e, "image_sample_c_b_cl">; +defm IMAGE_SAMPLE_C_LZ : MIMG_Sampler <0x0000002f, "image_sample_c_lz">; +defm IMAGE_SAMPLE_O : MIMG_Sampler_WQM <0x00000030, "image_sample_o">; +defm IMAGE_SAMPLE_CL_O : MIMG_Sampler_WQM <0x00000031, "image_sample_cl_o">; +defm IMAGE_SAMPLE_D_O : MIMG_Sampler <0x00000032, "image_sample_d_o">; +defm IMAGE_SAMPLE_D_CL_O : MIMG_Sampler <0x00000033, "image_sample_d_cl_o">; +defm IMAGE_SAMPLE_L_O : MIMG_Sampler <0x00000034, "image_sample_l_o">; +defm IMAGE_SAMPLE_B_O : MIMG_Sampler_WQM <0x00000035, "image_sample_b_o">; +defm IMAGE_SAMPLE_B_CL_O : MIMG_Sampler_WQM <0x00000036, "image_sample_b_cl_o">; +defm IMAGE_SAMPLE_LZ_O : MIMG_Sampler <0x00000037, "image_sample_lz_o">; +defm IMAGE_SAMPLE_C_O : MIMG_Sampler_WQM <0x00000038, "image_sample_c_o">; +defm IMAGE_SAMPLE_C_CL_O : MIMG_Sampler_WQM <0x00000039, "image_sample_c_cl_o">; +defm IMAGE_SAMPLE_C_D_O : MIMG_Sampler <0x0000003a, "image_sample_c_d_o">; +defm IMAGE_SAMPLE_C_D_CL_O : MIMG_Sampler <0x0000003b, "image_sample_c_d_cl_o">; +defm IMAGE_SAMPLE_C_L_O : MIMG_Sampler <0x0000003c, "image_sample_c_l_o">; +defm IMAGE_SAMPLE_C_B_O : MIMG_Sampler_WQM <0x0000003d, "image_sample_c_b_o">; +defm IMAGE_SAMPLE_C_B_CL_O : MIMG_Sampler_WQM <0x0000003e, "image_sample_c_b_cl_o">; +defm IMAGE_SAMPLE_C_LZ_O : MIMG_Sampler <0x0000003f, "image_sample_c_lz_o">; +defm IMAGE_GATHER4 : MIMG_Gather_WQM <0x00000040, "image_gather4">; +defm IMAGE_GATHER4_CL : MIMG_Gather_WQM <0x00000041, "image_gather4_cl">; +defm IMAGE_GATHER4_L : MIMG_Gather <0x00000044, "image_gather4_l">; +defm IMAGE_GATHER4_B : MIMG_Gather_WQM <0x00000045, "image_gather4_b">; +defm IMAGE_GATHER4_B_CL : MIMG_Gather_WQM <0x00000046, "image_gather4_b_cl">; +defm IMAGE_GATHER4_LZ : MIMG_Gather <0x00000047, "image_gather4_lz">; +defm IMAGE_GATHER4_C : MIMG_Gather_WQM <0x00000048, "image_gather4_c">; +defm IMAGE_GATHER4_C_CL : MIMG_Gather_WQM <0x00000049, "image_gather4_c_cl">; +defm IMAGE_GATHER4_C_L : MIMG_Gather <0x0000004c, "image_gather4_c_l">; +defm IMAGE_GATHER4_C_B : MIMG_Gather_WQM <0x0000004d, "image_gather4_c_b">; +defm IMAGE_GATHER4_C_B_CL : MIMG_Gather_WQM <0x0000004e, "image_gather4_c_b_cl">; +defm IMAGE_GATHER4_C_LZ : MIMG_Gather <0x0000004f, "image_gather4_c_lz">; +defm IMAGE_GATHER4_O : MIMG_Gather_WQM <0x00000050, "image_gather4_o">; +defm IMAGE_GATHER4_CL_O : MIMG_Gather_WQM <0x00000051, "image_gather4_cl_o">; +defm IMAGE_GATHER4_L_O : MIMG_Gather <0x00000054, "image_gather4_l_o">; +defm IMAGE_GATHER4_B_O : MIMG_Gather_WQM <0x00000055, "image_gather4_b_o">; +defm IMAGE_GATHER4_B_CL_O : MIMG_Gather <0x00000056, "image_gather4_b_cl_o">; +defm IMAGE_GATHER4_LZ_O : MIMG_Gather <0x00000057, "image_gather4_lz_o">; +defm IMAGE_GATHER4_C_O : MIMG_Gather_WQM <0x00000058, "image_gather4_c_o">; +defm IMAGE_GATHER4_C_CL_O : MIMG_Gather_WQM <0x00000059, "image_gather4_c_cl_o">; +defm IMAGE_GATHER4_C_L_O : MIMG_Gather <0x0000005c, "image_gather4_c_l_o">; +defm IMAGE_GATHER4_C_B_O : MIMG_Gather_WQM <0x0000005d, "image_gather4_c_b_o">; +defm IMAGE_GATHER4_C_B_CL_O : MIMG_Gather_WQM <0x0000005e, "image_gather4_c_b_cl_o">; +defm IMAGE_GATHER4_C_LZ_O : MIMG_Gather <0x0000005f, "image_gather4_c_lz_o">; +defm IMAGE_GET_LOD : MIMG_Sampler_WQM <0x00000060, "image_get_lod">; +defm IMAGE_SAMPLE_CD : MIMG_Sampler <0x00000068, "image_sample_cd">; +defm IMAGE_SAMPLE_CD_CL : MIMG_Sampler <0x00000069, "image_sample_cd_cl">; +defm IMAGE_SAMPLE_C_CD : MIMG_Sampler <0x0000006a, "image_sample_c_cd">; +defm IMAGE_SAMPLE_C_CD_CL : MIMG_Sampler <0x0000006b, "image_sample_c_cd_cl">; +defm IMAGE_SAMPLE_CD_O : MIMG_Sampler <0x0000006c, "image_sample_cd_o">; +defm IMAGE_SAMPLE_CD_CL_O : MIMG_Sampler <0x0000006d, "image_sample_cd_cl_o">; +defm IMAGE_SAMPLE_C_CD_O : MIMG_Sampler <0x0000006e, "image_sample_c_cd_o">; +defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler <0x0000006f, "image_sample_c_cd_cl_o">; +//def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", 0x0000007e>; +//def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", 0x0000007f>; +} + +/********** ======================= **********/ +/********** Image sampling patterns **********/ +/********** ======================= **********/ + +// Image + sampler +class SampleRawPattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, i32:$dmask, i32:$unorm, + i32:$r128, i32:$da, i32:$glc, i32:$slc, i32:$tfe, i32:$lwe), + (opcode $addr, $rsrc, $sampler, + (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), + (as_i1imm $r128), (as_i1imm $tfe), (as_i1imm $lwe), (as_i1imm $da)) +>; + +multiclass SampleRawPatterns { + def : SampleRawPattern(opcode # _V4_V1), i32>; + def : SampleRawPattern(opcode # _V4_V2), v2i32>; + def : SampleRawPattern(opcode # _V4_V4), v4i32>; + def : SampleRawPattern(opcode # _V4_V8), v8i32>; + def : SampleRawPattern(opcode # _V4_V16), v16i32>; +} + +// Image + sampler for amdgcn +// TODO: +// 1. Handle half data type like v4f16, and add D16 bit support; +// 2. Handle v4i32 rsrc type (Register Class for the instruction to be SReg_128). +// 3. Add A16 support when we pass address of half type. +multiclass AMDGCNSamplePattern { + def : Pat< + (v4f32 (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, i32:$dmask, i1:$unorm, i1:$glc, + i1:$slc, i1:$lwe, i1:$da)), + (opcode $addr, $rsrc, $sampler, + (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), + 0, 0, (as_i1imm $lwe), (as_i1imm $da)) + >; +} + +multiclass AMDGCNSamplePatterns { + defm : AMDGCNSamplePattern(opcode # _V4_V1), f32>; + defm : AMDGCNSamplePattern(opcode # _V4_V2), v2f32>; + defm : AMDGCNSamplePattern(opcode # _V4_V4), v4f32>; + defm : AMDGCNSamplePattern(opcode # _V4_V8), v8f32>; + defm : AMDGCNSamplePattern(opcode # _V4_V16), v16f32>; +} + +// Image only +class ImagePattern : Pat < + (name vt:$addr, v8i32:$rsrc, imm:$dmask, imm:$unorm, + imm:$r128, imm:$da, imm:$glc, imm:$slc, imm:$tfe, imm:$lwe), + (opcode $addr, $rsrc, + (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), + (as_i1imm $r128), (as_i1imm $tfe), (as_i1imm $lwe), (as_i1imm $da)) +>; + +multiclass ImagePatterns { + def : ImagePattern(opcode # _V4_V1), i32>; + def : ImagePattern(opcode # _V4_V2), v2i32>; + def : ImagePattern(opcode # _V4_V4), v4i32>; +} + +class ImageLoadPattern : Pat < + (name vt:$addr, v8i32:$rsrc, imm:$dmask, imm:$r128, imm:$da, imm:$glc, + imm:$slc), + (opcode $addr, $rsrc, + (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), + (as_i1imm $r128), 0, 0, (as_i1imm $da)) +>; + +multiclass ImageLoadPatterns { + def : ImageLoadPattern(opcode # _V4_V1), i32>; + def : ImageLoadPattern(opcode # _V4_V2), v2i32>; + def : ImageLoadPattern(opcode # _V4_V4), v4i32>; +} + +class ImageStorePattern : Pat < + (name v4f32:$data, vt:$addr, v8i32:$rsrc, i32:$dmask, imm:$r128, imm:$da, + imm:$glc, imm:$slc), + (opcode $data, $addr, $rsrc, + (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), + (as_i1imm $r128), 0, 0, (as_i1imm $da)) +>; + +multiclass ImageStorePatterns { + def : ImageStorePattern(opcode # _V4_V1), i32>; + def : ImageStorePattern(opcode # _V4_V2), v2i32>; + def : ImageStorePattern(opcode # _V4_V4), v4i32>; +} + +class ImageAtomicPattern : Pat < + (name i32:$vdata, vt:$addr, v8i32:$rsrc, imm:$r128, imm:$da, imm:$slc), + (opcode $vdata, $addr, $rsrc, 1, 1, 1, (as_i1imm $slc), (as_i1imm $r128), 0, 0, (as_i1imm $da)) +>; + +multiclass ImageAtomicPatterns { + def : ImageAtomicPattern(opcode # _V1), i32>; + def : ImageAtomicPattern(opcode # _V2), v2i32>; + def : ImageAtomicPattern(opcode # _V4), v4i32>; +} + +class ImageAtomicCmpSwapPattern : Pat < + (int_amdgcn_image_atomic_cmpswap i32:$vsrc, i32:$vcmp, vt:$addr, v8i32:$rsrc, + imm:$r128, imm:$da, imm:$slc), + (EXTRACT_SUBREG + (opcode (REG_SEQUENCE VReg_64, $vsrc, sub0, $vcmp, sub1), + $addr, $rsrc, 3, 1, 1, (as_i1imm $slc), (as_i1imm $r128), 0, 0, (as_i1imm $da)), + sub0) +>; + +// ======= SI Image Intrinsics ================ + +// Image load +defm : ImagePatterns; +defm : ImagePatterns; +def : ImagePattern; + +// Basic sample +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; + +// Sample with comparison +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; + +// Sample with offsets +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; + +// Sample with comparison and offsets +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; +defm : SampleRawPatterns; + +// Gather opcodes +// Only the variants which make sense are defined. +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; + +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; + +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; + +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; + +def : SampleRawPattern; +def : SampleRawPattern; +def : SampleRawPattern; + +// ======= amdgcn Image Intrinsics ============== + +// Image load +defm : ImageLoadPatterns; +defm : ImageLoadPatterns; + +// Image store +defm : ImageStorePatterns; +defm : ImageStorePatterns; + +// Basic sample +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; + +// Sample with comparison +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; + +// Sample with offsets +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; + +// Sample with comparison and offsets +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; +defm : AMDGCNSamplePatterns; + +// Gather opcodes +// Only the variants which make sense are defined. +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; + +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; + +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; + +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; + +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; +defm : AMDGCNSamplePattern; + +// Image atomics +defm : ImageAtomicPatterns; +def : ImageAtomicCmpSwapPattern; +def : ImageAtomicCmpSwapPattern; +def : ImageAtomicCmpSwapPattern; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; +defm : ImageAtomicPatterns; + +/* SIsample for simple 1D texture lookup */ +def : Pat < + (SIsample i32:$addr, v8i32:$rsrc, v4i32:$sampler, imm), + (IMAGE_SAMPLE_V4_V1 $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) +>; + +class SamplePattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, imm), + (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) +>; + +class SampleRectPattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_RECT), + (opcode $addr, $rsrc, $sampler, 0xf, 1, 0, 0, 0, 0, 0, 0) +>; + +class SampleArrayPattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_ARRAY), + (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 1) +>; + +class SampleShadowPattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_SHADOW), + (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) +>; + +class SampleShadowArrayPattern : Pat < + (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_SHADOW_ARRAY), + (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 1) +>; + +/* SIsample* for texture lookups consuming more address parameters */ +multiclass SamplePatterns { + def : SamplePattern ; + def : SampleRectPattern ; + def : SampleArrayPattern ; + def : SampleShadowPattern ; + def : SampleShadowArrayPattern ; + + def : SamplePattern ; + def : SampleArrayPattern ; + def : SampleShadowPattern ; + def : SampleShadowArrayPattern ; + + def : SamplePattern ; + def : SampleArrayPattern ; + def : SampleShadowPattern ; + def : SampleShadowArrayPattern ; + + def : SamplePattern ; + def : SampleArrayPattern ; + def : SampleShadowPattern ; + def : SampleShadowArrayPattern ; +} + +defm : SamplePatterns; +defm : SamplePatterns; +defm : SamplePatterns; +defm : SamplePatterns; Index: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td +++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.td @@ -2624,240 +2624,6 @@ } } -class MIMG_Mask { - string Op = op; - int Channels = channels; -} - -class mimg si, bits<7> vi = si> { - field bits<7> SI = si; - field bits<7> VI = vi; -} - -class MIMG_Helper : MIMG { - let mayLoad = 1; - let mayStore = 0; - let hasPostISelHook = 1; - let DecoderNamespace = dns; - let isAsmParserOnly = !if(!eq(dns,""), 1, 0); - let AsmMatchConverter = "cvtMIMG"; -} - -class MIMG_NoSampler_Helper op, string asm, - RegisterClass dst_rc, - RegisterClass addr_rc, - string dns=""> : MIMG_Helper < - (outs dst_rc:$vdata), - (ins addr_rc:$vaddr, SReg_256:$srsrc, - dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, - r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), - asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da", - dns>, MIMGe { - let ssamp = 0; -} - -multiclass MIMG_NoSampler_Src_Helper op, string asm, - RegisterClass dst_rc, - int channels> { - def _V1 : MIMG_NoSampler_Helper , - MIMG_Mask; - def _V2 : MIMG_NoSampler_Helper , - MIMG_Mask; - def _V4 : MIMG_NoSampler_Helper , - MIMG_Mask; -} - -multiclass MIMG_NoSampler op, string asm> { - defm _V1 : MIMG_NoSampler_Src_Helper ; - defm _V2 : MIMG_NoSampler_Src_Helper ; - defm _V3 : MIMG_NoSampler_Src_Helper ; - defm _V4 : MIMG_NoSampler_Src_Helper ; -} - -class MIMG_Store_Helper op, string asm, - RegisterClass data_rc, - RegisterClass addr_rc> : MIMG_Helper < - (outs), - (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, - dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, - r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), - asm#" $vdata, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" - >, MIMGe { - let ssamp = 0; - let mayLoad = 1; // TableGen requires this for matching with the intrinsics - let mayStore = 1; - let hasSideEffects = 1; - let hasPostISelHook = 0; - let DisableWQM = 1; -} - -multiclass MIMG_Store_Addr_Helper op, string asm, - RegisterClass data_rc, - int channels> { - def _V1 : MIMG_Store_Helper , - MIMG_Mask; - def _V2 : MIMG_Store_Helper , - MIMG_Mask; - def _V4 : MIMG_Store_Helper , - MIMG_Mask; -} - -multiclass MIMG_Store op, string asm> { - defm _V1 : MIMG_Store_Addr_Helper ; - defm _V2 : MIMG_Store_Addr_Helper ; - defm _V3 : MIMG_Store_Addr_Helper ; - defm _V4 : MIMG_Store_Addr_Helper ; -} - -class MIMG_Atomic_Helper : MIMG_Helper < - (outs data_rc:$vdst), - (ins data_rc:$vdata, addr_rc:$vaddr, SReg_256:$srsrc, - dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, - r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), - asm#" $vdst, $vaddr, $srsrc$dmask$unorm$glc$slc$r128$tfe$lwe$da" - > { - let mayStore = 1; - let hasSideEffects = 1; - let hasPostISelHook = 0; - let DisableWQM = 1; - let Constraints = "$vdst = $vdata"; - let AsmMatchConverter = "cvtMIMGAtomic"; -} - -class MIMG_Atomic_Real_si : - MIMG_Atomic_Helper, - SIMCInstr, - MIMGe { - let isCodeGenOnly = 0; - let AssemblerPredicates = [isSICI]; - let DecoderNamespace = "SICI"; - let DisableDecoder = DisableSIDecoder; -} - -class MIMG_Atomic_Real_vi : - MIMG_Atomic_Helper, - SIMCInstr, - MIMGe { - let isCodeGenOnly = 0; - let AssemblerPredicates = [isVI]; - let DecoderNamespace = "VI"; - let DisableDecoder = DisableVIDecoder; -} - -multiclass MIMG_Atomic_Helper_m { - let isPseudo = 1, isCodeGenOnly = 1 in { - def "" : MIMG_Atomic_Helper, - SIMCInstr; - } - - let ssamp = 0 in { - def _si : MIMG_Atomic_Real_si; - - def _vi : MIMG_Atomic_Real_vi; - } -} - -multiclass MIMG_Atomic { - defm _V1 : MIMG_Atomic_Helper_m ; - defm _V2 : MIMG_Atomic_Helper_m ; - defm _V4 : MIMG_Atomic_Helper_m ; -} - -class MIMG_Sampler_Helper op, string asm, - RegisterClass dst_rc, - RegisterClass src_rc, - int wqm, - string dns=""> : MIMG_Helper < - (outs dst_rc:$vdata), - (ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, - dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, - r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), - asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da", - dns>, MIMGe { - let WQM = wqm; -} - -multiclass MIMG_Sampler_Src_Helper op, string asm, - RegisterClass dst_rc, - int channels, int wqm> { - def _V1 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V2 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V4 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V8 : MIMG_Sampler_Helper , - MIMG_Mask; - def _V16 : MIMG_Sampler_Helper , - MIMG_Mask; -} - -multiclass MIMG_Sampler op, string asm, int wqm=0> { - defm _V1 : MIMG_Sampler_Src_Helper; - defm _V2 : MIMG_Sampler_Src_Helper; - defm _V3 : MIMG_Sampler_Src_Helper; - defm _V4 : MIMG_Sampler_Src_Helper; -} - -multiclass MIMG_Sampler_WQM op, string asm> : MIMG_Sampler; - -class MIMG_Gather_Helper op, string asm, - RegisterClass dst_rc, - RegisterClass src_rc, int wqm> : MIMG < - (outs dst_rc:$vdata), - (ins src_rc:$vaddr, SReg_256:$srsrc, SReg_128:$ssamp, - dmask:$dmask, unorm:$unorm, glc:$glc, slc:$slc, - r128:$r128, tfe:$tfe, lwe:$lwe, da:$da), - asm#" $vdata, $vaddr, $srsrc, $ssamp$dmask$unorm$glc$slc$r128$tfe$lwe$da", - []>, MIMGe { - let mayLoad = 1; - let mayStore = 0; - - // DMASK was repurposed for GATHER4. 4 components are always - // returned and DMASK works like a swizzle - it selects - // the component to fetch. The only useful DMASK values are - // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns - // (red,red,red,red) etc.) The ISA document doesn't mention - // this. - // Therefore, disable all code which updates DMASK by setting this: - let Gather4 = 1; - let hasPostISelHook = 0; - let WQM = wqm; - - let isAsmParserOnly = 1; // TBD: fix it later -} - -multiclass MIMG_Gather_Src_Helper op, string asm, - RegisterClass dst_rc, - int channels, int wqm> { - def _V1 : MIMG_Gather_Helper , - MIMG_Mask; - def _V2 : MIMG_Gather_Helper , - MIMG_Mask; - def _V4 : MIMG_Gather_Helper , - MIMG_Mask; - def _V8 : MIMG_Gather_Helper , - MIMG_Mask; - def _V16 : MIMG_Gather_Helper , - MIMG_Mask; -} - -multiclass MIMG_Gather op, string asm, int wqm=0> { - defm _V1 : MIMG_Gather_Src_Helper; - defm _V2 : MIMG_Gather_Src_Helper; - defm _V3 : MIMG_Gather_Src_Helper; - defm _V4 : MIMG_Gather_Src_Helper; -} - -multiclass MIMG_Gather_WQM op, string asm> : MIMG_Gather; - //===----------------------------------------------------------------------===// // Vector instruction mappings //===----------------------------------------------------------------------===// @@ -2964,3 +2730,4 @@ include "VIInstructions.td" include "DSInstructions.td" +include "MIMGInstructions.td" Index: llvm/trunk/lib/Target/AMDGPU/SIInstructions.td =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIInstructions.td +++ llvm/trunk/lib/Target/AMDGPU/SIInstructions.td @@ -448,106 +448,6 @@ defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Store_Helper <0x00000007, "tbuffer_store_format_xyzw", VReg_128>; //===----------------------------------------------------------------------===// -// MIMG Instructions -//===----------------------------------------------------------------------===// - -defm IMAGE_LOAD : MIMG_NoSampler <0x00000000, "image_load">; -defm IMAGE_LOAD_MIP : MIMG_NoSampler <0x00000001, "image_load_mip">; -//def IMAGE_LOAD_PCK : MIMG_NoPattern_ <"image_load_pck", 0x00000002>; -//def IMAGE_LOAD_PCK_SGN : MIMG_NoPattern_ <"image_load_pck_sgn", 0x00000003>; -//def IMAGE_LOAD_MIP_PCK : MIMG_NoPattern_ <"image_load_mip_pck", 0x00000004>; -//def IMAGE_LOAD_MIP_PCK_SGN : MIMG_NoPattern_ <"image_load_mip_pck_sgn", 0x00000005>; -defm IMAGE_STORE : MIMG_Store <0x00000008, "image_store">; -defm IMAGE_STORE_MIP : MIMG_Store <0x00000009, "image_store_mip">; -//def IMAGE_STORE_PCK : MIMG_NoPattern_ <"image_store_pck", 0x0000000a>; -//def IMAGE_STORE_MIP_PCK : MIMG_NoPattern_ <"image_store_mip_pck", 0x0000000b>; -defm IMAGE_GET_RESINFO : MIMG_NoSampler <0x0000000e, "image_get_resinfo">; -defm IMAGE_ATOMIC_SWAP : MIMG_Atomic , "image_atomic_swap">; -defm IMAGE_ATOMIC_CMPSWAP : MIMG_Atomic , "image_atomic_cmpswap", VReg_64>; -defm IMAGE_ATOMIC_ADD : MIMG_Atomic , "image_atomic_add">; -defm IMAGE_ATOMIC_SUB : MIMG_Atomic , "image_atomic_sub">; -//def IMAGE_ATOMIC_RSUB : MIMG_NoPattern_ <"image_atomic_rsub", 0x00000013>; -- not on VI -defm IMAGE_ATOMIC_SMIN : MIMG_Atomic , "image_atomic_smin">; -defm IMAGE_ATOMIC_UMIN : MIMG_Atomic , "image_atomic_umin">; -defm IMAGE_ATOMIC_SMAX : MIMG_Atomic , "image_atomic_smax">; -defm IMAGE_ATOMIC_UMAX : MIMG_Atomic , "image_atomic_umax">; -defm IMAGE_ATOMIC_AND : MIMG_Atomic , "image_atomic_and">; -defm IMAGE_ATOMIC_OR : MIMG_Atomic , "image_atomic_or">; -defm IMAGE_ATOMIC_XOR : MIMG_Atomic , "image_atomic_xor">; -defm IMAGE_ATOMIC_INC : MIMG_Atomic , "image_atomic_inc">; -defm IMAGE_ATOMIC_DEC : MIMG_Atomic , "image_atomic_dec">; -//def IMAGE_ATOMIC_FCMPSWAP : MIMG_NoPattern_ <"image_atomic_fcmpswap", 0x0000001d>; -- not on VI -//def IMAGE_ATOMIC_FMIN : MIMG_NoPattern_ <"image_atomic_fmin", 0x0000001e>; -- not on VI -//def IMAGE_ATOMIC_FMAX : MIMG_NoPattern_ <"image_atomic_fmax", 0x0000001f>; -- not on VI -defm IMAGE_SAMPLE : MIMG_Sampler_WQM <0x00000020, "image_sample">; -defm IMAGE_SAMPLE_CL : MIMG_Sampler_WQM <0x00000021, "image_sample_cl">; -defm IMAGE_SAMPLE_D : MIMG_Sampler <0x00000022, "image_sample_d">; -defm IMAGE_SAMPLE_D_CL : MIMG_Sampler <0x00000023, "image_sample_d_cl">; -defm IMAGE_SAMPLE_L : MIMG_Sampler <0x00000024, "image_sample_l">; -defm IMAGE_SAMPLE_B : MIMG_Sampler_WQM <0x00000025, "image_sample_b">; -defm IMAGE_SAMPLE_B_CL : MIMG_Sampler_WQM <0x00000026, "image_sample_b_cl">; -defm IMAGE_SAMPLE_LZ : MIMG_Sampler <0x00000027, "image_sample_lz">; -defm IMAGE_SAMPLE_C : MIMG_Sampler_WQM <0x00000028, "image_sample_c">; -defm IMAGE_SAMPLE_C_CL : MIMG_Sampler_WQM <0x00000029, "image_sample_c_cl">; -defm IMAGE_SAMPLE_C_D : MIMG_Sampler <0x0000002a, "image_sample_c_d">; -defm IMAGE_SAMPLE_C_D_CL : MIMG_Sampler <0x0000002b, "image_sample_c_d_cl">; -defm IMAGE_SAMPLE_C_L : MIMG_Sampler <0x0000002c, "image_sample_c_l">; -defm IMAGE_SAMPLE_C_B : MIMG_Sampler_WQM <0x0000002d, "image_sample_c_b">; -defm IMAGE_SAMPLE_C_B_CL : MIMG_Sampler_WQM <0x0000002e, "image_sample_c_b_cl">; -defm IMAGE_SAMPLE_C_LZ : MIMG_Sampler <0x0000002f, "image_sample_c_lz">; -defm IMAGE_SAMPLE_O : MIMG_Sampler_WQM <0x00000030, "image_sample_o">; -defm IMAGE_SAMPLE_CL_O : MIMG_Sampler_WQM <0x00000031, "image_sample_cl_o">; -defm IMAGE_SAMPLE_D_O : MIMG_Sampler <0x00000032, "image_sample_d_o">; -defm IMAGE_SAMPLE_D_CL_O : MIMG_Sampler <0x00000033, "image_sample_d_cl_o">; -defm IMAGE_SAMPLE_L_O : MIMG_Sampler <0x00000034, "image_sample_l_o">; -defm IMAGE_SAMPLE_B_O : MIMG_Sampler_WQM <0x00000035, "image_sample_b_o">; -defm IMAGE_SAMPLE_B_CL_O : MIMG_Sampler_WQM <0x00000036, "image_sample_b_cl_o">; -defm IMAGE_SAMPLE_LZ_O : MIMG_Sampler <0x00000037, "image_sample_lz_o">; -defm IMAGE_SAMPLE_C_O : MIMG_Sampler_WQM <0x00000038, "image_sample_c_o">; -defm IMAGE_SAMPLE_C_CL_O : MIMG_Sampler_WQM <0x00000039, "image_sample_c_cl_o">; -defm IMAGE_SAMPLE_C_D_O : MIMG_Sampler <0x0000003a, "image_sample_c_d_o">; -defm IMAGE_SAMPLE_C_D_CL_O : MIMG_Sampler <0x0000003b, "image_sample_c_d_cl_o">; -defm IMAGE_SAMPLE_C_L_O : MIMG_Sampler <0x0000003c, "image_sample_c_l_o">; -defm IMAGE_SAMPLE_C_B_O : MIMG_Sampler_WQM <0x0000003d, "image_sample_c_b_o">; -defm IMAGE_SAMPLE_C_B_CL_O : MIMG_Sampler_WQM <0x0000003e, "image_sample_c_b_cl_o">; -defm IMAGE_SAMPLE_C_LZ_O : MIMG_Sampler <0x0000003f, "image_sample_c_lz_o">; -defm IMAGE_GATHER4 : MIMG_Gather_WQM <0x00000040, "image_gather4">; -defm IMAGE_GATHER4_CL : MIMG_Gather_WQM <0x00000041, "image_gather4_cl">; -defm IMAGE_GATHER4_L : MIMG_Gather <0x00000044, "image_gather4_l">; -defm IMAGE_GATHER4_B : MIMG_Gather_WQM <0x00000045, "image_gather4_b">; -defm IMAGE_GATHER4_B_CL : MIMG_Gather_WQM <0x00000046, "image_gather4_b_cl">; -defm IMAGE_GATHER4_LZ : MIMG_Gather <0x00000047, "image_gather4_lz">; -defm IMAGE_GATHER4_C : MIMG_Gather_WQM <0x00000048, "image_gather4_c">; -defm IMAGE_GATHER4_C_CL : MIMG_Gather_WQM <0x00000049, "image_gather4_c_cl">; -defm IMAGE_GATHER4_C_L : MIMG_Gather <0x0000004c, "image_gather4_c_l">; -defm IMAGE_GATHER4_C_B : MIMG_Gather_WQM <0x0000004d, "image_gather4_c_b">; -defm IMAGE_GATHER4_C_B_CL : MIMG_Gather_WQM <0x0000004e, "image_gather4_c_b_cl">; -defm IMAGE_GATHER4_C_LZ : MIMG_Gather <0x0000004f, "image_gather4_c_lz">; -defm IMAGE_GATHER4_O : MIMG_Gather_WQM <0x00000050, "image_gather4_o">; -defm IMAGE_GATHER4_CL_O : MIMG_Gather_WQM <0x00000051, "image_gather4_cl_o">; -defm IMAGE_GATHER4_L_O : MIMG_Gather <0x00000054, "image_gather4_l_o">; -defm IMAGE_GATHER4_B_O : MIMG_Gather_WQM <0x00000055, "image_gather4_b_o">; -defm IMAGE_GATHER4_B_CL_O : MIMG_Gather <0x00000056, "image_gather4_b_cl_o">; -defm IMAGE_GATHER4_LZ_O : MIMG_Gather <0x00000057, "image_gather4_lz_o">; -defm IMAGE_GATHER4_C_O : MIMG_Gather_WQM <0x00000058, "image_gather4_c_o">; -defm IMAGE_GATHER4_C_CL_O : MIMG_Gather_WQM <0x00000059, "image_gather4_c_cl_o">; -defm IMAGE_GATHER4_C_L_O : MIMG_Gather <0x0000005c, "image_gather4_c_l_o">; -defm IMAGE_GATHER4_C_B_O : MIMG_Gather_WQM <0x0000005d, "image_gather4_c_b_o">; -defm IMAGE_GATHER4_C_B_CL_O : MIMG_Gather_WQM <0x0000005e, "image_gather4_c_b_cl_o">; -defm IMAGE_GATHER4_C_LZ_O : MIMG_Gather <0x0000005f, "image_gather4_c_lz_o">; -defm IMAGE_GET_LOD : MIMG_Sampler_WQM <0x00000060, "image_get_lod">; -defm IMAGE_SAMPLE_CD : MIMG_Sampler <0x00000068, "image_sample_cd">; -defm IMAGE_SAMPLE_CD_CL : MIMG_Sampler <0x00000069, "image_sample_cd_cl">; -defm IMAGE_SAMPLE_C_CD : MIMG_Sampler <0x0000006a, "image_sample_c_cd">; -defm IMAGE_SAMPLE_C_CD_CL : MIMG_Sampler <0x0000006b, "image_sample_c_cd_cl">; -defm IMAGE_SAMPLE_CD_O : MIMG_Sampler <0x0000006c, "image_sample_cd_o">; -defm IMAGE_SAMPLE_CD_CL_O : MIMG_Sampler <0x0000006d, "image_sample_cd_cl_o">; -defm IMAGE_SAMPLE_C_CD_O : MIMG_Sampler <0x0000006e, "image_sample_c_cd_o">; -defm IMAGE_SAMPLE_C_CD_CL_O : MIMG_Sampler <0x0000006f, "image_sample_c_cd_cl_o">; -//def IMAGE_RSRC256 : MIMG_NoPattern_RSRC256 <"image_rsrc256", 0x0000007e>; -//def IMAGE_SAMPLER : MIMG_NoPattern_ <"image_sampler", 0x0000007f>; - -//===----------------------------------------------------------------------===// // VOP1 Instructions //===----------------------------------------------------------------------===// @@ -1833,416 +1733,6 @@ $src2_modifiers, $src2, $clamp, $omod) >; -/********** ======================= **********/ -/********** Image sampling patterns **********/ -/********** ======================= **********/ - -// Image + sampler -class SampleRawPattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, i32:$dmask, i32:$unorm, - i32:$r128, i32:$da, i32:$glc, i32:$slc, i32:$tfe, i32:$lwe), - (opcode $addr, $rsrc, $sampler, - (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), - (as_i1imm $r128), (as_i1imm $tfe), (as_i1imm $lwe), (as_i1imm $da)) ->; - -multiclass SampleRawPatterns { - def : SampleRawPattern(opcode # _V4_V1), i32>; - def : SampleRawPattern(opcode # _V4_V2), v2i32>; - def : SampleRawPattern(opcode # _V4_V4), v4i32>; - def : SampleRawPattern(opcode # _V4_V8), v8i32>; - def : SampleRawPattern(opcode # _V4_V16), v16i32>; -} - - -// Image + sampler for amdgcn -// TODO: -// 1. Handle half data type like v4f16, and add D16 bit support; -// 2. Handle v4i32 rsrc type (Register Class for the instruction to be SReg_128). -// 3. Add A16 support when we pass address of half type. -multiclass AMDGCNSamplePattern { - def : Pat< - (v4f32 (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, i32:$dmask, i1:$unorm, i1:$glc, - i1:$slc, i1:$lwe, i1:$da)), - (opcode $addr, $rsrc, $sampler, - (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), - 0, 0, (as_i1imm $lwe), (as_i1imm $da)) - >; -} - -multiclass AMDGCNSamplePatterns { - defm : AMDGCNSamplePattern(opcode # _V4_V1), f32>; - defm : AMDGCNSamplePattern(opcode # _V4_V2), v2f32>; - defm : AMDGCNSamplePattern(opcode # _V4_V4), v4f32>; - defm : AMDGCNSamplePattern(opcode # _V4_V8), v8f32>; - defm : AMDGCNSamplePattern(opcode # _V4_V16), v16f32>; -} - - -// Image only -class ImagePattern : Pat < - (name vt:$addr, v8i32:$rsrc, imm:$dmask, imm:$unorm, - imm:$r128, imm:$da, imm:$glc, imm:$slc, imm:$tfe, imm:$lwe), - (opcode $addr, $rsrc, - (as_i32imm $dmask), (as_i1imm $unorm), (as_i1imm $glc), (as_i1imm $slc), - (as_i1imm $r128), (as_i1imm $tfe), (as_i1imm $lwe), (as_i1imm $da)) ->; - -multiclass ImagePatterns { - def : ImagePattern(opcode # _V4_V1), i32>; - def : ImagePattern(opcode # _V4_V2), v2i32>; - def : ImagePattern(opcode # _V4_V4), v4i32>; -} - -class ImageLoadPattern : Pat < - (name vt:$addr, v8i32:$rsrc, imm:$dmask, imm:$r128, imm:$da, imm:$glc, - imm:$slc), - (opcode $addr, $rsrc, - (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), - (as_i1imm $r128), 0, 0, (as_i1imm $da)) ->; - -multiclass ImageLoadPatterns { - def : ImageLoadPattern(opcode # _V4_V1), i32>; - def : ImageLoadPattern(opcode # _V4_V2), v2i32>; - def : ImageLoadPattern(opcode # _V4_V4), v4i32>; -} - -class ImageStorePattern : Pat < - (name v4f32:$data, vt:$addr, v8i32:$rsrc, i32:$dmask, imm:$r128, imm:$da, - imm:$glc, imm:$slc), - (opcode $data, $addr, $rsrc, - (as_i32imm $dmask), 1, (as_i1imm $glc), (as_i1imm $slc), - (as_i1imm $r128), 0, 0, (as_i1imm $da)) ->; - -multiclass ImageStorePatterns { - def : ImageStorePattern(opcode # _V4_V1), i32>; - def : ImageStorePattern(opcode # _V4_V2), v2i32>; - def : ImageStorePattern(opcode # _V4_V4), v4i32>; -} - -class ImageAtomicPattern : Pat < - (name i32:$vdata, vt:$addr, v8i32:$rsrc, imm:$r128, imm:$da, imm:$slc), - (opcode $vdata, $addr, $rsrc, 1, 1, 1, (as_i1imm $slc), (as_i1imm $r128), 0, 0, (as_i1imm $da)) ->; - -multiclass ImageAtomicPatterns { - def : ImageAtomicPattern(opcode # _V1), i32>; - def : ImageAtomicPattern(opcode # _V2), v2i32>; - def : ImageAtomicPattern(opcode # _V4), v4i32>; -} - -class ImageAtomicCmpSwapPattern : Pat < - (int_amdgcn_image_atomic_cmpswap i32:$vsrc, i32:$vcmp, vt:$addr, v8i32:$rsrc, - imm:$r128, imm:$da, imm:$slc), - (EXTRACT_SUBREG - (opcode (REG_SEQUENCE VReg_64, $vsrc, sub0, $vcmp, sub1), - $addr, $rsrc, 3, 1, 1, (as_i1imm $slc), (as_i1imm $r128), 0, 0, (as_i1imm $da)), - sub0) ->; - -// ======= SI Image Intrinsics ================ - -// Image load -defm : ImagePatterns; -defm : ImagePatterns; -def : ImagePattern; - -// Basic sample -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; - -// Sample with comparison -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; - -// Sample with offsets -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; - -// Sample with comparison and offsets -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; -defm : SampleRawPatterns; - -// Gather opcodes -// Only the variants which make sense are defined. -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; - -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; - -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; - -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; - -def : SampleRawPattern; -def : SampleRawPattern; -def : SampleRawPattern; - - -// ======= amdgcn Image Intrinsics ============== - -// Image load -defm : ImageLoadPatterns; -defm : ImageLoadPatterns; - -// Image store -defm : ImageStorePatterns; -defm : ImageStorePatterns; - -// Basic sample -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; - -// Sample with comparison -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; - -// Sample with offsets -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; - -// Sample with comparison and offsets -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; -defm : AMDGCNSamplePatterns; - -// Gather opcodes -// Only the variants which make sense are defined. -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; - -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; - -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; - -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; - -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; -defm : AMDGCNSamplePattern; - -// Image atomics -defm : ImageAtomicPatterns; -def : ImageAtomicCmpSwapPattern; -def : ImageAtomicCmpSwapPattern; -def : ImageAtomicCmpSwapPattern; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; -defm : ImageAtomicPatterns; - -/* SIsample for simple 1D texture lookup */ -def : Pat < - (SIsample i32:$addr, v8i32:$rsrc, v4i32:$sampler, imm), - (IMAGE_SAMPLE_V4_V1 $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) ->; - -class SamplePattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, imm), - (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) ->; - -class SampleRectPattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_RECT), - (opcode $addr, $rsrc, $sampler, 0xf, 1, 0, 0, 0, 0, 0, 0) ->; - -class SampleArrayPattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_ARRAY), - (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 1) ->; - -class SampleShadowPattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_SHADOW), - (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 0) ->; - -class SampleShadowArrayPattern : Pat < - (name vt:$addr, v8i32:$rsrc, v4i32:$sampler, TEX_SHADOW_ARRAY), - (opcode $addr, $rsrc, $sampler, 0xf, 0, 0, 0, 0, 0, 0, 1) ->; - -/* SIsample* for texture lookups consuming more address parameters */ -multiclass SamplePatterns { - def : SamplePattern ; - def : SampleRectPattern ; - def : SampleArrayPattern ; - def : SampleShadowPattern ; - def : SampleShadowArrayPattern ; - - def : SamplePattern ; - def : SampleArrayPattern ; - def : SampleShadowPattern ; - def : SampleShadowArrayPattern ; - - def : SamplePattern ; - def : SampleArrayPattern ; - def : SampleShadowPattern ; - def : SampleShadowArrayPattern ; - - def : SamplePattern ; - def : SampleArrayPattern ; - def : SampleShadowPattern ; - def : SampleShadowArrayPattern ; -} - -defm : SamplePatterns; -defm : SamplePatterns; -defm : SamplePatterns; -defm : SamplePatterns; - /********** ============================================ **********/ /********** Extraction, Insertion, Building and Casting **********/ /********** ============================================ **********/