[AMDGPU] Properly mark MUBUF and FLAT LDS DMA instructions. NFC.

[rampitec]

Summary

Add these bits to the MUBUF and FLAT LDS DMA instructions:

• LGKM_CNT - these operate on LDS;
• VALU - SPG 3.9.8: This instruction acts as both a MUBUF and

VALU instruction;

Codegen currently does not produce any of this, so the change is NFC.

[arsenm]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

Loads and stores shouldn't be marked hasSideEffects, only mayLoad/mayStore

[rampitec]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

How do you suggest to mark that it can read and write virtually any memory?
Note that atomics have hasSideEffects flag.

[arsenm]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

They shouldn't have it set either. mayLoad or mayStore indicate this. the MMO may just not have meaningful pointer info

[arsenm]

Probably should fix any atomics marked as hasSideEffests

[foad]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

So these pseudos should either have 0 MMOs (no info, conservatively correct) or 2 MMOs (one for the BUF access and one for the LDS). Having just 1 MMO would be wrong.

[rampitec]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

They should have 2 (I have no idea how to to distinguish load and store for a DMA). But the reality is I cannot express this operation in terms of pointers and ranges. This is scatter on the load side and gather on the store side. Or vice versa for the opposite operation.

[foad]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

Right, but you can have an MMO with the appropriate address space, even if it has no pointer range. That still provides some info to alias analysis.

[rampitec]

llvm/lib/Target/AMDGPU/BUFInstructions.td

518

I am not particularly good in discussing a Schroedinger cat. This does not exist since we do not produce it in IR. There is no way you can get it in MIR.

If you really want to think about this start with the hazard recognizer and how does it bail after classifying an instruction to be a VALU or *BUF. That is what I am about now.

[arsenm]

This should have 2 separate MMOs, and the verifier should probably enforce that if we were to start producing this

[rampitec]

In D124472#3477559, @arsenm wrote:

This should have 2 separate MMOs, and the verifier should probably enforce that if we were to start producing this

Unfortunately wait count pass uses MMO pointers, so we have to do it. I can imagine a base pointer, but I have no idea how much memory does this touch. At the very least it depends on the WG size.

[arsenm]

In D124472#3478927, @rampitec wrote:

In D124472#3477559, @arsenm wrote:

This should have 2 separate MMOs, and the verifier should probably enforce that if we were to start producing this

Unfortunately wait count pass uses MMO pointers, so we have to do it. I can imagine a base pointer, but I have no idea how much memory does this touch. At the very least it depends on the WG size.

MMOs can set the size to unknown-size

[rampitec]

Intersting that I can only create such MMOs during lowering. I do not see how an intrinsic call in IR can get 2 distinct MMOs. I have managed to get one, which is a 'load store' on address space 4. This is most certainly not a normal memory operation for llvm and not even a normal memory intrinsic. I cannot really express that an intrinsic reads addrspace 1 and writes addrspace 3. I think I have even figured out how much memory does it read and write, but the data type in too wide to express: wave size * operation size. For a single dword that is v64i32, but it can be 4 times more.

llvm/lib/Target/AMDGPU/BUFInstructions.td

class MUBUF_Real <MUBUF_Pseudo ps> :
   let DisableEncoding      = ps.DisableEncoding;
   let TSFlags              = ps.TSFlags;
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let SchedRW              = ps.SchedRW;
   let mayLoad              = ps.mayLoad;
   let mayStore             = ps.mayStore;
   let IsAtomicRet          = ps.IsAtomicRet;
   let IsAtomicNoRet        = ps.IsAtomicNoRet;
+  let VALU                 = ps.VALU;
+  let LGKM_CNT             = ps.LGKM_CNT;
 
   bits<12> offset;
   bits<5>  cpol;
   bits<8>  vaddr;
   bits<10> vdata;
   bits<7>  srsrc;
   bits<1>  tfe;
   bits<8>  soffset;
: MUBUF_Pseudo<opName,
                    !if(isLds, " lds", "$tfe") # "$swz",
                  pattern>,
     MUBUF_SetupAddr<addrKindCopy> {
   let PseudoInstr = opName # !if(isLds, "_lds", "") #
                     "_" # getAddrName<addrKindCopy>.ret;
   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 
   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
+  let LGKM_CNT = isLds;
   let mayLoad = 1;
   let mayStore = 0;
   let maybeAtomic = 1;
   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
   let has_tfe = !not(isLds);
   let lds = isLds;
   let elements = getMUBUFElements<vdata_vt>.ret;
+  let VALU = isLds;
 }
 
 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))),
   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset))
 >;
 
 class MUBUF_Addr64_Load_Pat <Instruction inst,
                             ValueType load_vt = i32,
multiclass MUBUF_Pseudo_Stores<string opName,
   }
 }
 
 class MUBUF_Pseudo_Store_Lds<string opName>
   : MUBUF_Pseudo<opName,
                  (outs),
                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz),
                  " $srsrc, $soffset$offset lds$cpol$swz"> {
+  let LGKM_CNT = 1;
   let mayLoad = 0;
   let mayStore = 1;
   let maybeAtomic = 1;
 
   let has_vdata = 0;
   let has_vaddr = 0;
   let has_tfe = 0;
   let lds = 1;
+  let VALU = 1;
 
   let Uses = [EXEC, M0];
   let AsmMatchConverter = "cvtMubufLds";
 }
 
 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
                           list<RegisterClass> vaddrList=[]> {
   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));

llvm/lib/Target/AMDGPU/FLATInstructions.td

class FLAT_Real <bits<7> op, FLAT_Pseudo ps> :
   let UseNamedOperandTable = ps.UseNamedOperandTable;
   let SchedRW              = ps.SchedRW;
   let mayLoad              = ps.mayLoad;
   let mayStore             = ps.mayStore;
   let IsAtomicRet          = ps.IsAtomicRet;
   let IsAtomicNoRet        = ps.IsAtomicNoRet;
   let VM_CNT               = ps.VM_CNT;
   let LGKM_CNT             = ps.LGKM_CNT;
+  let VALU                 = ps.VALU;
 
   // encoding fields
   bits<8> vaddr;
   bits<10> vdata;
   bits<7> saddr;
   bits<10> vdst;
 
   bits<5> cpol;
class FLAT_Global_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0> : FLAT_Pseudo<
   let LGKM_CNT = 1;
   let is_flat_global = 1;
   let has_data = 0;
   let has_vdst = 0;
   let mayLoad = 1;
   let mayStore = 1;
   let has_saddr = 1;
   let enabled_saddr = EnableSaddr;
+  let VALU = 1;
   let PseudoInstr = opName#!if(EnableSaddr, "_SADDR", "");
   let Uses = [M0, EXEC];
   let SchedRW = [WriteVMEM, WriteLDS];
 }
 
 multiclass FLAT_Global_Load_LDS_Pseudo<string opName> {
   def ""     : FLAT_Global_Load_LDS_Pseudo<opName>,
     GlobalSaddrTable<0, opName>;
class FLAT_Scratch_Load_LDS_Pseudo <string opName, bit EnableSaddr = 0,
   let has_vdst = 0;
   let mayLoad = 1;
   let mayStore = 1;
   let has_saddr = 1;
   let enabled_saddr = EnableSaddr;
   let has_vaddr = EnableVaddr;
   let has_sve = EnableSVE;
   let sve = EnableVaddr;
+  let VALU = 1;
   let PseudoInstr = opName#!if(EnableSVE, "_SVS", !if(EnableSaddr, "_SADDR", !if(EnableVaddr, "", "_ST")));
   let Uses = [M0, EXEC];
   let SchedRW = [WriteVMEM, WriteLDS];
 }
 
 multiclass FLAT_Scratch_Load_LDS_Pseudo<string opName> {
   def ""     : FLAT_Scratch_Load_LDS_Pseudo<opName>,
                FlatScratchInst<opName, "SV">;