AMDGPU: Add definitions for scalar store instructions

[arsenm]

Summary

Also add glc bit to the scalar loads since they exist on VI
and change the caching behavior.

This currently has an assembler bug where the glc bit is incorrectly
accepted on SI/CI which do not have it.

[• tstellarAMD]

LGTM.

[arsenm]

Fix available type. Add verifier check that offset SGPR is m0

[arsenm]

r285463

lib/Target/AMDGPU/AMDGPU.td

 >;
 
 def FeatureVGPRIndexMode : SubtargetFeature<"vgpr-index-mode",
   "HasVGPRIndexMode",
   "true",
   "Has VGPR mode register indexing"
 >;
 
+def FeatureScalarStores : SubtargetFeature<"scalar-stores",
+  "HasScalarStores",
+  "true",
+  "Has store scalar memory instructions"
+>;
+
 //===------------------------------------------------------------===//
 // Subtarget Features (options and debugging)
 //===------------------------------------------------------------===//
 
 // Some instructions do not support denormals despite this flag. Using
 // fp32 denormals also causes instructions to run at the double
 // precision rate for the device.
 def FeatureFP32Denormals : SubtargetFeature<"fp32-denormals",
def FeatureSeaIslands : SubtargetFeatureGeneration<"SEA_ISLANDS",
   FeatureGCN1Encoding, FeatureCIInsts, FeatureMovrel]
 >;
 
 def FeatureVolcanicIslands : SubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
   [FeatureFP64, FeatureLocalMemorySize65536,
    FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
    FeatureGCN3Encoding, FeatureCIInsts, FeatureMovrel,
    Feature16BitInsts, FeatureInv2PiInlineImm, FeatureSMemRealTime,
-   FeatureVGPRIndexMode
+   FeatureVGPRIndexMode, FeatureScalarStores
   ]
 >;
 
 //===----------------------------------------------------------------------===//
 // Debugger related subtarget features.
 //===----------------------------------------------------------------------===//
 
 def FeatureDebuggerInsertNops : SubtargetFeature<

lib/Target/AMDGPU/AMDGPUSubtarget.h

protected:
   bool GCN3Encoding;
   bool CIInsts;
   bool SGPRInitBug;
   bool HasSMemRealTime;
   bool Has16BitInsts;
   bool HasMovrel;
   bool HasVGPRIndexMode;
   bool HasInv2PiInlineImm;
+  bool HasScalarStores;
   bool FlatAddressSpace;
   bool R600ALUInst;
   bool CaymanISA;
   bool CFALUBug;
   bool HasVertexCache;
   short TexVTXClauseSize;
 
   // Dummy feature to use for assembler in tablegen.
public:
   bool hasScalarCompareEq64() const {
     return getGeneration() >= VOLCANIC_ISLANDS;
   }
 
   bool hasInv2PiInlineImm() const {
     return HasInv2PiInlineImm;
   }
 
+  bool hasScalarStores() const {
+    return HasScalarStores;
+  }
+
   bool enableSIScheduler() const {
     return EnableSIScheduler;
   }
 
   bool debuggerSupported() const {
     return debuggerInsertNops() && debuggerReserveRegs() &&
       debuggerEmitPrologue();
   }

lib/Target/AMDGPU/SIDefines.h

enum {
   MIMG = 1 << 20,
   FLAT = 1 << 21,
   WQM = 1 << 22,
   VGPRSpill = 1 << 23,
   SGPRSpill = 1 << 24,
   VOPAsmPrefer32Bit = 1 << 25,
   Gather4 = 1 << 26,
   DisableWQM = 1 << 27,
-  SOPK_ZEXT = 1 << 28
+  SOPK_ZEXT = 1 << 28,
+  SCALAR_STORE = 1 << 29
 };
 }
 
 namespace llvm {
 namespace AMDGPU {
   enum OperandType {
     /// Operands with register or 32-bit immediate
     OPERAND_REG_IMM32_INT = MCOI::OPERAND_FIRST_TARGET,

lib/Target/AMDGPU/SIInstrFormats.td

class InstSI <dag outs, dag ins, string asm = "",
 
   // Whether WQM _must_ be disabled for this instruction.
   field bits<1> DisableWQM = 0;
 
   // Most sopk treat the immediate as a signed 16-bit, however some
   // use it as unsigned.
   field bits<1> SOPKZext = 0;
 
+  // This is an s_store_dword* instruction that requires a cache flush
+  // on wave termination. It is necessary to distinguish from mayStore
+  // SMEM instructions like the cache flush ones.
+  field bits<1> ScalarStore = 0;
+
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = VM_CNT;
   let TSFlags{1} = EXP_CNT;
   let TSFlags{2} = LGKM_CNT;
 
   let TSFlags{3} = SALU;
   let TSFlags{4} = VALU;
 
class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{21} = FLAT;
   let TSFlags{22} = WQM;
   let TSFlags{23} = VGPRSpill;
   let TSFlags{24} = SGPRSpill;
   let TSFlags{25} = VOPAsmPrefer32Bit;
   let TSFlags{26} = Gather4;
   let TSFlags{27} = DisableWQM;
   let TSFlags{28} = SOPKZext;
+  let TSFlags{29} = ScalarStore;
 
   let SchedRW = [Write32Bit];
 
   field bits<1> DisableSIDecoder = 0;
   field bits<1> DisableVIDecoder = 0;
   field bits<1> DisableDecoder = 0;
 
   let isAsmParserOnly = !if(!eq(DisableDecoder{0}, {0}), 0, 1);

lib/Target/AMDGPU/SIInstrInfo.h

public:
   static bool sopkIsZext(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::SOPK_ZEXT;
   }
 
   bool sopkIsZext(uint16_t Opcode) const {
     return get(Opcode).TSFlags & SIInstrFlags::SOPK_ZEXT;
   }
 
+  /// \returns true if this is an s_store_dword* instruction. This is more
+  /// specific than than isSMEM && mayStore.
+  static bool isScalarStore(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;
+  }
+
+  bool isScalarStore(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;
+  }
+
   bool isVGPRCopy(const MachineInstr &MI) const {
     assert(MI.isCopy());
     unsigned Dest = MI.getOperand(0).getReg();
     const MachineFunction &MF = *MI.getParent()->getParent();
     const MachineRegisterInfo &MRI = MF.getRegInfo();
     return !RI.isSGPRReg(MRI, Dest);
   }
 

lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
   // being careful when using let Uses to try to add other use registers.
   if (shouldReadExec(MI)) {
     if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
       ErrInfo = "VALU instruction does not implicitly read exec mask";
       return false;
     }
   }
 
+  if (isSMRD(MI)) {
+    if (MI.mayStore()) {
+      // The register offset form of scalar stores may only use m0 as the
+      // soffset register.
+      const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
+      if (Soff && Soff->getReg() != AMDGPU::M0) {
+        ErrInfo = "scalar stores must use m0 as offset register";
+        return false;
+      }
+    }
+  }
+
   return true;
 }
 
 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default: return AMDGPU::INSTRUCTION_LIST_END;
   case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
   case AMDGPU::COPY: return AMDGPU::COPY;

lib/Target/AMDGPU/SMInstructions.td

class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
   let SchedRW = [WriteSMEM];
   let SubtargetPredicate = isGCN;
 
   string Mnemonic = opName;
   string AsmOperands = asmOps;
 
   bits<1> has_sbase = 1;
   bits<1> has_sdst = 1;
+  bit has_glc = 0;
   bits<1> has_offset = 1;
   bits<1> offset_is_imm = 0;
 }
 
 class SM_Real <SM_Pseudo ps>
   : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 
   let isPseudo = 0;
   let isCodeGenOnly = 0;
 
   // copy relevant pseudo op flags
   let SubtargetPredicate = ps.SubtargetPredicate;
   let AsmMatchConverter  = ps.AsmMatchConverter;
 
   // encoding
   bits<7>  sbase;
   bits<7>  sdst;
   bits<32> offset;
-  bits<1>  imm = !if(ps.has_offset, ps.offset_is_imm, 0);
+  bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0);
 }
 
 class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]>
   : SM_Pseudo<opName, outs, ins, asmOps, pattern> {
   RegisterClass BaseClass;
+  let mayLoad = 1;
+  let mayStore = 0;
+  let has_glc = 1;
+}
+
+class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []>
+  : SM_Pseudo<opName, (outs), ins, asmOps, pattern> {
+  RegisterClass BaseClass;
+  RegisterClass SrcClass;
+  let mayLoad = 0;
+  let mayStore = 1;
+  let has_glc = 1;
+  let ScalarStore = 1;
 }
 
 multiclass SM_Pseudo_Loads<string opName,
                            RegisterClass baseClass,
                            RegisterClass dstClass> {
   def _IMM  : SM_Load_Pseudo <opName,
                               (outs dstClass:$sdst),
-                              (ins baseClass:$sbase, i32imm:$offset),
-                              " $sdst, $sbase, $offset", []> {
+                              (ins baseClass:$sbase, i32imm:$offset, i1imm:$glc),
+                              " $sdst, $sbase, $offset$glc", []> {
     let offset_is_imm = 1;
     let BaseClass = baseClass;
     let PseudoInstr = opName # "_IMM";
+    let has_glc = 1;
   }
+
   def _SGPR  : SM_Load_Pseudo <opName,
                               (outs dstClass:$sdst),
-                              (ins baseClass:$sbase, SReg_32:$soff),
-                              " $sdst, $sbase, $offset", []> {
+                              (ins baseClass:$sbase, SReg_32:$soff, i1imm:$glc),
+                              " $sdst, $sbase, $offset$glc", []> {
+    let BaseClass = baseClass;
+    let PseudoInstr = opName # "_SGPR";
+    let has_glc = 1;
+  }
+}
+
+multiclass SM_Pseudo_Stores<string opName,
+                           RegisterClass baseClass,
+                           RegisterClass srcClass> {
+  def _IMM  : SM_Store_Pseudo <opName,
+    (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, i1imm:$glc),
+    " $sdata, $sbase, $offset$glc", []> {
+    let offset_is_imm = 1;
+    let BaseClass = baseClass;
+    let SrcClass = srcClass;
+    let PseudoInstr = opName # "_IMM";
+  }
+
+  def _SGPR  : SM_Store_Pseudo <opName,
+    (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, i1imm:$glc),
+    " $sdata, $sbase, $offset$glc", []> {
     let BaseClass = baseClass;
+    let SrcClass = srcClass;
     let PseudoInstr = opName # "_SGPR";
   }
 }
 
 class SM_Time_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
   opName, (outs SReg_64:$sdst), (ins),
   " $sdst", [(set i64:$sdst, (node))]> {
   let hasSideEffects = 1;

 defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx8", SReg_128, SReg_256
 >;
 
 defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <
   "s_buffer_load_dwordx16", SReg_128, SReg_512
 >;
 
+defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0>;
+defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64>;
+defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>;
+
+defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <
+  "s_buffer_store_dword", SReg_128, SReg_32_XM0
+>;
+
+defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <
+  "s_buffer_store_dwordx2", SReg_128, SReg_64
+>;
+
+defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <
+  "s_buffer_store_dwordx4", SReg_128, SReg_128
+>;
+
+
 def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
 def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
 
 let SubtargetPredicate = isCIVI in {
 def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
 } // let SubtargetPredicate = isCIVI
 
 let SubtargetPredicate = isVI in {

 
 let Predicates = [isGCN] in {
 
 multiclass SMRD_Pattern <string Instr, ValueType vt> {
 
   // 1. IMM offset
   def : Pat <
     (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset))
+    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
   >;
 
   // 2. SGPR offset
   def : Pat <
     (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)),
-    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset))
+    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0))
   >;
 }
 
 let Predicates = [isSICI] in {
 def : Pat <
   (i64 (readcyclecounter)),
   (S_MEMTIME)
 >;
 }
 
 // Global and constant loads can be selected to either MUBUF or SMRD
 // instructions, but SMRD instructions are faster so we want the instruction
 // selector to prefer those.
 let AddedComplexity = 100 in {
 
 defm : SMRD_Pattern <"S_LOAD_DWORD",    i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX2",  v2i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX4",  v4i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX8",  v8i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
 
 // 1. Offset as an immediate
 def SM_LOAD_PATTERN : Pat <  // name this pattern to reuse AddedComplexity on CI
   (SIload_constant v4i32:$sbase, (SMRDBufferImm i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset)
+  (S_BUFFER_LOAD_DWORD_IMM $sbase, $offset, 0)
 >;
 
 // 2. Offset loaded in an 32bit SGPR
 def : Pat <
   (SIload_constant v4i32:$sbase, (SMRDBufferSgpr i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset)
+  (S_BUFFER_LOAD_DWORD_SGPR $sbase, $offset, 0)
 >;
 
 } // End let AddedComplexity = 100
 
 } // let Predicates = [isGCN]
 
 let Predicates = [isVI] in {
 
 // 1. Offset as 20bit DWORD immediate
 def : Pat <
   (SIload_constant v4i32:$sbase, IMM20bit:$offset),
-  (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset))
+  (S_BUFFER_LOAD_DWORD_IMM $sbase, (as_i32imm $offset), 0)
 >;
 
 def : Pat <
   (i64 (readcyclecounter)),
   (S_MEMREALTIME)
 >;
 
 } // let Predicates = [isVI]
class SMRD_Real_si <bits<5> op, SM_Pseudo ps>
   let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, ?);
   let Inst{8}     = imm;
   let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
   let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
   let Inst{26-22} = op;
   let Inst{31-27} = 0x18; //encoding
 }
 
+// FIXME: Assembler should reject trying to use glc on SMRD
+// instructions on SI.
 multiclass SM_Real_Loads_si<bits<5> op, string ps,
                             SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
                             SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
+
   def _IMM_si : SMRD_Real_si <op, immPs> {
-    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset);
+    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset, GLC:$glc);
   }
+
+  // FIXME: The operand name $offset is inconsistent with $soff used
+  // in the pseudo
   def _SGPR_si : SMRD_Real_si <op, sgprPs> {
-    let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset);
+    let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
   }
+
 }
 
 defm S_LOAD_DWORD           : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">;
 defm S_LOAD_DWORDX2         : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">;
 defm S_LOAD_DWORDX4         : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">;
 defm S_LOAD_DWORDX8         : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">;
 defm S_LOAD_DWORDX16        : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">;
 defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">;

 //===----------------------------------------------------------------------===//
 // VI
 //===----------------------------------------------------------------------===//
 
 class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
   : SM_Real<ps>
   , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI>
   , Enc64 {
+  bit glc;
 
   let AssemblerPredicates = [isVI];
   let DecoderNamespace = "VI";
 
   let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
   let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
 
-  // glc is only applicable to scalar stores, which are not yet
-  // implemented.
-  let Inst{16}    = 0; // glc bit
-  let Inst{17}    = imm;
+  let Inst{16} = !if(ps.has_glc, glc, ?);
+  let Inst{17} = imm;
   let Inst{25-18} = op;
   let Inst{31-26} = 0x30; //encoding
   let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?);
 }
 
 multiclass SM_Real_Loads_vi<bits<8> op, string ps,
                             SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
                             SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
   def _IMM_vi : SMEM_Real_vi <op, immPs> {
-    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset);
-  }
+    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset:$offset, GLC:$glc);
+  }
+  def _SGPR_vi : SMEM_Real_vi <op, sgprPs> {
+    let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
+  }
+
+multiclass SM_Real_Stores_vi<bits<8> op, string ps,
+                            SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
+  // FIXME: The operand name $offset is inconsistent with $soff used
+  // in the pseudo
   def _SGPR_vi : SMEM_Real_vi <op, sgprPs> {
-    let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset);
+    let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
   }
 }
 
 defm S_LOAD_DWORD           : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">;
 defm S_LOAD_DWORDX2         : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">;
 defm S_LOAD_DWORDX4         : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">;
 defm S_LOAD_DWORDX8         : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">;
 defm S_LOAD_DWORDX16        : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">;
 defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">;
 defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">;
 defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">;
 defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">;
 defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">;
 
+defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">;
+defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">;
+defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">;
+
+defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">;
+defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">;
+defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">;
+
 def S_DCACHE_INV_vi         : SMEM_Real_vi <0x20, S_DCACHE_INV>;
 def S_DCACHE_WB_vi          : SMEM_Real_vi <0x21, S_DCACHE_WB>;
 def S_DCACHE_INV_VOL_vi     : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;
 def S_DCACHE_WB_VOL_vi      : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;
 def S_MEMTIME_vi            : SMEM_Real_vi <0x24, S_MEMTIME>;
 def S_MEMREALTIME_vi        : SMEM_Real_vi <0x25, S_MEMREALTIME>;
 
 
 //===----------------------------------------------------------------------===//
 // CI
 //===----------------------------------------------------------------------===//
 
 def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset",
                                           NamedMatchClass<"SMRDLiteralOffset">> {
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
 class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :
   SM_Real<ps>,
   Enc64 {
 
   let AssemblerPredicates = [isCIOnly];
   let DecoderNamespace = "CI";
-  let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset);
+  let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc);
 
   let LGKM_CNT = ps.LGKM_CNT;
   let SMRD = ps.SMRD;
   let mayLoad = ps.mayLoad;
   let mayStore = ps.mayStore;
   let hasSideEffects = ps.hasSideEffects;
   let SchedRW = ps.SchedRW;
   let UseNamedOperandTable = ps.UseNamedOperandTable;

 }
 
 def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
 
 let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity in {
 
 class SMRD_Pattern_ci <string Instr, ValueType vt> : Pat <
   (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
-  (vt (!cast<SM_Pseudo>(Instr#"_IMM_ci") $sbase, $offset))> {
+  (vt (!cast<SM_Pseudo>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
   let Predicates = [isCIOnly];
 }
 
 def : SMRD_Pattern_ci <"S_LOAD_DWORD",    i32>;
 def : SMRD_Pattern_ci <"S_LOAD_DWORDX2",  v2i32>;
 def : SMRD_Pattern_ci <"S_LOAD_DWORDX4",  v4i32>;
 def : SMRD_Pattern_ci <"S_LOAD_DWORDX8",  v8i32>;
 def : SMRD_Pattern_ci <"S_LOAD_DWORDX16", v16i32>;
 
 def : Pat <
   (SIload_constant v4i32:$sbase, (SMRDBufferImm32 i32:$offset)),
-  (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset)> {
+  (S_BUFFER_LOAD_DWORD_IMM_ci $sbase, $offset, 0)> {
   let Predicates = [isCI]; // should this be isCIOnly?
 }
 
 } // End let AddedComplexity = SM_LOAD_PATTERN.AddedComplexity
 

test/CodeGen/AMDGPU/coalescer-subreg-join.mir

frameInfo:
   hasMustTailInVarArgFunc: false
 body:             |
   bb.0:
     liveins: %sgpr2_sgpr3, %vgpr2, %vgpr3
 
     %0 = COPY %sgpr2_sgpr3
     %1 = COPY %vgpr2
     %2 = COPY %vgpr3
-    %3 = S_LOAD_DWORDX8_IMM %0, 0
-    %4 = S_LOAD_DWORDX4_IMM %0, 12
-    %5 = S_LOAD_DWORDX8_IMM %0, 16
-    %6 = S_LOAD_DWORDX4_IMM %0, 28
+    %3 = S_LOAD_DWORDX8_IMM %0, 0, 0
+    %4 = S_LOAD_DWORDX4_IMM %0, 12, 0
+    %5 = S_LOAD_DWORDX8_IMM %0, 16, 0
+    %6 = S_LOAD_DWORDX4_IMM %0, 28, 0
     undef %7.sub0 = S_MOV_B32 212739
     %20 = COPY %7
     %11 = COPY %20
     %11.sub1 = COPY %1
     %11.sub2 = COPY %1
     %11.sub3 = COPY %1
     %11.sub4 = COPY %1
     %11.sub5 = COPY %1

test/CodeGen/MIR/AMDGPU/target-index-operands.mir

-# RUN: llc -march=amdgcn -mcpu=SI -run-pass none -o - %s | FileCheck %s
+# RUN: llc -march=amdgcn -run-pass none -o - %s | FileCheck %s
 # This test verifies that the MIR parser can parse target index operands.
 
 --- |
 
   %struct.foo = type { float, [5 x i32] }
 
   @float_gv = internal unnamed_addr addrspace(2) constant [5 x float] [float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00], align 4
 
body: |
   bb.0.entry:
     liveins: %sgpr0_sgpr1
 
     %sgpr2_sgpr3 = S_GETPC_B64
   ; CHECK: %sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start), implicit-def %scc, implicit-def %scc
     %sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start), implicit-def %scc, implicit-def %scc
     %sgpr3 = S_ADDC_U32 %sgpr3, 0, implicit-def %scc, implicit %scc, implicit-def %scc, implicit %scc
     %sgpr4_sgpr5 = S_LSHR_B64 %sgpr2_sgpr3, 32, implicit-def dead %scc
-    %sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11
+    %sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11, 0
     %sgpr7 = S_ASHR_I32 %sgpr6, 31, implicit-def dead %scc
     %sgpr6_sgpr7 = S_LSHL_B64 %sgpr6_sgpr7, 2, implicit-def dead %scc
     %sgpr2 = S_ADD_U32 %sgpr2, @float_gv, implicit-def %scc
     %sgpr3 = S_ADDC_U32 %sgpr4, 0, implicit-def dead %scc, implicit %scc
     %sgpr4 = S_ADD_U32 %sgpr2, %sgpr6, implicit-def %scc
     %sgpr5 = S_ADDC_U32 %sgpr3, %sgpr7, implicit-def dead %scc, implicit %scc
-    %sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0
-    %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0, 0
+    %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9, 0
     %sgpr7 = S_MOV_B32 61440
     %sgpr6 = S_MOV_B32 -1
     %vgpr0 = V_MOV_B32_e32 killed %sgpr2, implicit %exec
     BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec
     S_ENDPGM
 ...
 ---
 name: float2
 liveins:
   - { reg: '%sgpr0_sgpr1' }
 frameInfo:
   maxAlignment:  8
 body: |
   bb.0.entry:
     liveins: %sgpr0_sgpr1
 
     %sgpr2_sgpr3 = S_GETPC_B64
   ; CHECK: %sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start) + 1, implicit-def %scc, implicit-def %scc
     %sgpr2 = S_ADD_U32 %sgpr2, target-index(amdgpu-constdata-start) + 1, implicit-def %scc, implicit-def %scc
     %sgpr3 = S_ADDC_U32 %sgpr3, 0, implicit-def %scc, implicit %scc, implicit-def %scc, implicit %scc
     %sgpr4_sgpr5 = S_LSHR_B64 %sgpr2_sgpr3, 32, implicit-def dead %scc
-    %sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11
+    %sgpr6 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 11, 0
     %sgpr7 = S_ASHR_I32 %sgpr6, 31, implicit-def dead %scc
     %sgpr6_sgpr7 = S_LSHL_B64 %sgpr6_sgpr7, 2, implicit-def dead %scc
     %sgpr2 = S_ADD_U32 %sgpr2, @float_gv, implicit-def %scc
     %sgpr3 = S_ADDC_U32 %sgpr4, 0, implicit-def dead %scc, implicit %scc
     %sgpr4 = S_ADD_U32 %sgpr2, %sgpr6, implicit-def %scc
     %sgpr5 = S_ADDC_U32 %sgpr3, %sgpr7, implicit-def dead %scc, implicit %scc
-    %sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0
-    %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr4_sgpr5, 0, 0
+    %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9, 0
     %sgpr7 = S_MOV_B32 61440
     %sgpr6 = S_MOV_B32 -1
     %vgpr0 = V_MOV_B32_e32 killed %sgpr2, implicit %exec
     BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec
     S_ENDPGM
 ...

test/MC/AMDGPU/smem.s

 // RUN: llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 // RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti %s 2>&1 | FileCheck -check-prefix=NOSI %s
 // RUN: not llvm-mc -arch=amdgcn -mcpu=bonaire %s 2>&1 | FileCheck -check-prefix=NOSI %s
 
 s_dcache_wb
 // VI: s_dcache_wb  ; encoding: [0x00,0x00,0x84,0xc0,0x00,0x00,0x00,0x00]
 // NOSI: error: instruction not supported on this GPU
 
 s_dcache_wb_vol
 // VI: s_dcache_wb_vol ; encoding: [0x00,0x00,0x8c,0xc0,0x00,0x00,0x00,0x00]
 // NOSI: error: instruction not supported on this GPU
 
 s_memrealtime s[4:5]
 // VI: s_memrealtime s[4:5] ; encoding: [0x00,0x01,0x94,0xc0,0x00,0x00,0x00,0x00]
 // NOSI: error: instruction not supported on this GPU
+
+// FIXME: Should error about instruction on GPU
+s_store_dword s1, s[2:3], 0xfc
+// VI: s_store_dword s1, s[2:3], 0xfc  ; encoding: [0x41,0x00,0x42,0xc0,0xfc,0x00,0x00,0x00]
+// NOSI: error: instruction not supported on this GPU
+
+s_store_dword s1, s[2:3], 0xfc glc
+// VI: s_store_dword s1, s[2:3], 0xfc glc ; encoding: [0x41,0x00,0x43,0xc0,0xfc,0x00,0x00,0x00]
+// NOSI: error: invalid operand for instruction
+
+s_store_dword s1, s[2:3], s4
+// VI: s_store_dword s1, s[2:3], s4    ; encoding: [0x41,0x00,0x40,0xc0,0x04,0x00,0x00,0x00]
+// NOSI: error: instruction not supported on this GPU
+
+s_store_dword s1, s[2:3], s4 glc
+// VI: s_store_dword s1, s[2:3], s4 glc ; encoding: [0x41,0x00,0x41,0xc0,0x04,0x00,0x00,0x00]
+// NOSI: error: invalid operand for instruction
+
+// FIXME: Should error on SI instead of silently ignoring glc
+s_load_dword s1, s[2:3], 0xfc glc
+// VI: s_load_dword s1, s[2:3], 0xfc glc ; encoding: [0x41,0x00,0x03,0xc0,0xfc,0x00,0x00,0x00]
+
+s_load_dword s1, s[2:3], s4 glc
+// VI: s_load_dword s1, s[2:3], s4 glc ; encoding: [0x41,0x00,0x01,0xc0,0x04,0x00,0x00,0x00]

test/MC/AMDGPU/smrd-err.s

-// RUN: llvm-mc -arch=amdgcn -mcpu=tahiti %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti %s 2>&1 | FileCheck -check-prefix=GCN -check-prefix=SI %s
 // RUN: not llvm-mc -arch=amdgcn -mcpu=tonga %s 2>&1 | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 s_load_dwordx4 s[100:103], s[2:3], s4
 // VI: error: not a valid operand
 // SI: s_load_dwordx4 s[100:103], s[2:3], s4
 
 
 s_load_dwordx8 s[96:103], s[2:3], s4
 // VI: error: not a valid operand
 // SI:  s_load_dwordx8 s[96:103], s[2:3], s4
 
 s_load_dwordx16 s[88:103], s[2:3], s4
 // VI: error: not a valid operand
 // SI: s_load_dwordx16 s[88:103], s[2:3], s4