[AMDGPU][MC] Enable lds_direct operand for v_readfirstlane_b32, v_readlane_b32 and v_writelane_b32

[dp]

Summary

See bug 40662: https://bugs.llvm.org/show_bug.cgi?id=40662

Note that v_writelane_b32 accepts lds_direct for gfx7 only.

This fix also disables lds_direct for all 'rev' opcodes (not in docs, but confirmed by HW people).

[rampitec]

LGTM

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

public:
   bool isSCSrcF32() const {
     return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
   }
 
   bool isSCSrcF64() const {
     return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
   }
 
+  bool isSCSrcOrLdsB32() const {
+    return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32);
+  }
+
   bool isSSrcB32() const {
     return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
   }
 
   bool isSSrcB16() const {
     return isSCSrcB16() || isLiteralImm(MVT::i16);
   }
 
bool isSSrcF16() const {
     return isSCSrcB16() || isLiteralImm(MVT::f16);
   }
 
   bool isSSrcV2F16() const {
     llvm_unreachable("cannot happen");
     return isSSrcF16();
   }
 
+  bool isSSrcOrLdsB32() const {
+    return isSCSrcOrLdsB32() || isLiteralImm(MVT::i32) || isExpr();
+  }
+
   bool isVCSrcB32() const {
     return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
   }
 
   bool isVCSrcB64() const {
     return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
   }
 
bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
   if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm()) {
     if (isCI() || isSI())
       return false;
   }
 
   return true;
 }
 
+static bool IsRevOpcode(const unsigned Opcode)
+{
+  switch (Opcode) {
+  case AMDGPU::V_SUBREV_F32_e32:
+  case AMDGPU::V_SUBREV_F32_e64:
+  case AMDGPU::V_SUBREV_F32_e32_si:
+  case AMDGPU::V_SUBREV_F32_e32_vi:
+  case AMDGPU::V_SUBREV_F32_e64_si:
+  case AMDGPU::V_SUBREV_F32_e64_vi:
+  case AMDGPU::V_SUBREV_I32_e32:
+  case AMDGPU::V_SUBREV_I32_e64:
+  case AMDGPU::V_SUBREV_I32_e32_si:
+  case AMDGPU::V_SUBREV_I32_e64_si:
+  case AMDGPU::V_SUBBREV_U32_e32:
+  case AMDGPU::V_SUBBREV_U32_e64:
+  case AMDGPU::V_SUBBREV_U32_e32_si:
+  case AMDGPU::V_SUBBREV_U32_e32_vi:
+  case AMDGPU::V_SUBBREV_U32_e64_si:
+  case AMDGPU::V_SUBBREV_U32_e64_vi:
+  case AMDGPU::V_SUBREV_U32_e32:
+  case AMDGPU::V_SUBREV_U32_e64:
+  case AMDGPU::V_SUBREV_U32_e32_gfx9:
+  case AMDGPU::V_SUBREV_U32_e32_vi:
+  case AMDGPU::V_SUBREV_U32_e64_gfx9:
+  case AMDGPU::V_SUBREV_U32_e64_vi:
+  case AMDGPU::V_SUBREV_F16_e32:
+  case AMDGPU::V_SUBREV_F16_e64:
+  case AMDGPU::V_SUBREV_F16_e32_vi:
+  case AMDGPU::V_SUBREV_F16_e64_vi:
+  case AMDGPU::V_SUBREV_U16_e32:
+  case AMDGPU::V_SUBREV_U16_e64:
+  case AMDGPU::V_SUBREV_U16_e32_vi:
+  case AMDGPU::V_SUBREV_U16_e64_vi:
+  case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
+  case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
+  case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
+  case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
+  case AMDGPU::V_LSHLREV_B32_e32_si:
+  case AMDGPU::V_LSHLREV_B32_e64_si:
+  case AMDGPU::V_LSHLREV_B16_e32_vi:
+  case AMDGPU::V_LSHLREV_B16_e64_vi:
+  case AMDGPU::V_LSHLREV_B32_e32_vi:
+  case AMDGPU::V_LSHLREV_B32_e64_vi:
+  case AMDGPU::V_LSHLREV_B64_vi:
+  case AMDGPU::V_LSHRREV_B32_e32_si:
+  case AMDGPU::V_LSHRREV_B32_e64_si:
+  case AMDGPU::V_LSHRREV_B16_e32_vi:
+  case AMDGPU::V_LSHRREV_B16_e64_vi:
+  case AMDGPU::V_LSHRREV_B32_e32_vi:
+  case AMDGPU::V_LSHRREV_B32_e64_vi:
+  case AMDGPU::V_LSHRREV_B64_vi:
+  case AMDGPU::V_ASHRREV_I32_e64_si:
+  case AMDGPU::V_ASHRREV_I32_e32_si:
+  case AMDGPU::V_ASHRREV_I16_e32_vi:
+  case AMDGPU::V_ASHRREV_I16_e64_vi:
+  case AMDGPU::V_ASHRREV_I32_e32_vi:
+  case AMDGPU::V_ASHRREV_I32_e64_vi:
+  case AMDGPU::V_ASHRREV_I64_vi:
+  case AMDGPU::V_PK_LSHLREV_B16_vi:
+  case AMDGPU::V_PK_LSHRREV_B16_vi:
+  case AMDGPU::V_PK_ASHRREV_I16_vi:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
 
   using namespace SIInstrFlags;
   const unsigned Opcode = Inst.getOpcode();
   const MCInstrDesc &Desc = MII.get(Opcode);
 
   // lds_direct register is defined so that it can be used
   // with 9-bit operands only. Ignore encodings which do not accept these.
bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
   if (Src0Idx == -1)
     return true;
 
   const MCOperand &Src = Inst.getOperand(Src0Idx);
   if (!Src.isReg() || Src.getReg() != LDS_DIRECT)
     return true;
 
   // lds_direct is specified as src0. Check additional limitations.
-
-  // FIXME: This is a workaround for bug 37943
-  // which allows 64-bit VOP3 opcodes use 32-bit operands.
-  if (AMDGPU::getRegOperandSize(getMRI(), Desc, Src0Idx) != 4)
-    return false;
-
-  // Documentation does not disable lds_direct for SDWA, but SP3 assembler does.
-  // FIXME: This inconsistence needs to be investigated further.
-  if (Desc.TSFlags & SIInstrFlags::SDWA)
-    return false;
-
-  // The following opcodes do not accept lds_direct which is explicitly stated
-  // in AMD documentation. However SP3 disables lds_direct for most other 'rev'
-  // opcodes as well (e.g. for v_subrev_u32 but not for v_subrev_f32).
-  // FIXME: This inconsistence needs to be investigated further.
-  switch (Opcode) {
-  case AMDGPU::V_LSHLREV_B32_e32_si:
-  case AMDGPU::V_LSHLREV_B32_e64_si:
-  case AMDGPU::V_LSHLREV_B16_e32_vi:
-  case AMDGPU::V_LSHLREV_B16_e64_vi:
-  case AMDGPU::V_LSHLREV_B32_e32_vi:
-  case AMDGPU::V_LSHLREV_B32_e64_vi:
-  case AMDGPU::V_LSHLREV_B64_vi:
-  case AMDGPU::V_LSHRREV_B32_e32_si:
-  case AMDGPU::V_LSHRREV_B32_e64_si:
-  case AMDGPU::V_LSHRREV_B16_e32_vi:
-  case AMDGPU::V_LSHRREV_B16_e64_vi:
-  case AMDGPU::V_LSHRREV_B32_e32_vi:
-  case AMDGPU::V_LSHRREV_B32_e64_vi:
-  case AMDGPU::V_LSHRREV_B64_vi:
-  case AMDGPU::V_ASHRREV_I32_e64_si:
-  case AMDGPU::V_ASHRREV_I32_e32_si:
-  case AMDGPU::V_ASHRREV_I16_e32_vi:
-  case AMDGPU::V_ASHRREV_I16_e64_vi:
-  case AMDGPU::V_ASHRREV_I32_e32_vi:
-  case AMDGPU::V_ASHRREV_I32_e64_vi:
-  case AMDGPU::V_ASHRREV_I64_vi:
-  case AMDGPU::V_PK_LSHLREV_B16_vi:
-  case AMDGPU::V_PK_LSHRREV_B16_vi:
-  case AMDGPU::V_PK_ASHRREV_I16_vi:
-    return false;
-  default:
-    return true;
-  }
+  return (Desc.TSFlags & SIInstrFlags::SDWA) == 0 && !IsRevOpcode(Opcode);
 }
 
 bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
   unsigned Opcode = Inst.getOpcode();
   const MCInstrDesc &Desc = MII.get(Opcode);
   if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
     return true;
 

lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h

public:
 
   DecodeStatus tryDecodeInst(const uint8_t* Table, MCInst &MI, uint64_t Inst,
                              uint64_t Address) const;
 
   DecodeStatus convertSDWAInst(MCInst &MI) const;
   DecodeStatus convertMIMGInst(MCInst &MI) const;
 
   MCOperand decodeOperand_VGPR_32(unsigned Val) const;
+  MCOperand decodeOperand_VRegOrLds_32(unsigned Val) const;
+
   MCOperand decodeOperand_VS_32(unsigned Val) const;
   MCOperand decodeOperand_VS_64(unsigned Val) const;
   MCOperand decodeOperand_VS_128(unsigned Val) const;
   MCOperand decodeOperand_VSrc16(unsigned Val) const;
   MCOperand decodeOperand_VSrcV216(unsigned Val) const;
 
   MCOperand decodeOperand_VReg_64(unsigned Val) const;
   MCOperand decodeOperand_VReg_96(unsigned Val) const;
   MCOperand decodeOperand_VReg_128(unsigned Val) const;
 
   MCOperand decodeOperand_SReg_32(unsigned Val) const;
   MCOperand decodeOperand_SReg_32_XM0_XEXEC(unsigned Val) const;
   MCOperand decodeOperand_SReg_32_XEXEC_HI(unsigned Val) const;
+  MCOperand decodeOperand_SRegOrLds_32(unsigned Val) const;
   MCOperand decodeOperand_SReg_64(unsigned Val) const;
   MCOperand decodeOperand_SReg_64_XEXEC(unsigned Val) const;
   MCOperand decodeOperand_SReg_128(unsigned Val) const;
   MCOperand decodeOperand_SReg_256(unsigned Val) const;
   MCOperand decodeOperand_SReg_512(unsigned Val) const;
 
   enum OpWidthTy {
     OPW32,

lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

static DecodeStatus StaticDecoderName(MCInst &Inst, \
   auto DAsm = static_cast<const AMDGPUDisassembler*>(Decoder); \
   return addOperand(Inst, DAsm->DecoderName(Imm)); \
 }
 
 #define DECODE_OPERAND_REG(RegClass) \
 DECODE_OPERAND(Decode##RegClass##RegisterClass, decodeOperand_##RegClass)
 
 DECODE_OPERAND_REG(VGPR_32)
+DECODE_OPERAND_REG(VRegOrLds_32)
 DECODE_OPERAND_REG(VS_32)
 DECODE_OPERAND_REG(VS_64)
 DECODE_OPERAND_REG(VS_128)
 
 DECODE_OPERAND_REG(VReg_64)
 DECODE_OPERAND_REG(VReg_96)
 DECODE_OPERAND_REG(VReg_128)
 
 DECODE_OPERAND_REG(SReg_32)
 DECODE_OPERAND_REG(SReg_32_XM0_XEXEC)
 DECODE_OPERAND_REG(SReg_32_XEXEC_HI)
+DECODE_OPERAND_REG(SRegOrLds_32)
 DECODE_OPERAND_REG(SReg_64)
 DECODE_OPERAND_REG(SReg_64_XEXEC)
 DECODE_OPERAND_REG(SReg_128)
 DECODE_OPERAND_REG(SReg_256)
 DECODE_OPERAND_REG(SReg_512)
 
 static DecodeStatus decodeOperand_VSrc16(MCInst &Inst,
                                          unsigned Imm,
MCOperand AMDGPUDisassembler::decodeOperand_VGPR_32(unsigned Val) const {
   // Some instructions have operand restrictions beyond what the encoding
   // allows. Some ordinarily VSrc_32 operands are VGPR_32, so clear the extra
   // high bit.
   Val &= 255;
 
   return createRegOperand(AMDGPU::VGPR_32RegClassID, Val);
 }
 
+MCOperand AMDGPUDisassembler::decodeOperand_VRegOrLds_32(unsigned Val) const {
+  return decodeSrcOp(OPW32, Val);
+}
+
 MCOperand AMDGPUDisassembler::decodeOperand_VReg_64(unsigned Val) const {
   return createRegOperand(AMDGPU::VReg_64RegClassID, Val);
 }
 
 MCOperand AMDGPUDisassembler::decodeOperand_VReg_96(unsigned Val) const {
   return createRegOperand(AMDGPU::VReg_96RegClassID, Val);
 }
 

 }
 
 MCOperand AMDGPUDisassembler::decodeOperand_SReg_32_XEXEC_HI(
   unsigned Val) const {
   // SReg_32_XM0 is SReg_32 without EXEC_HI
   return decodeOperand_SReg_32(Val);
 }
 
+MCOperand AMDGPUDisassembler::decodeOperand_SRegOrLds_32(unsigned Val) const {
+  // table-gen generated disassembler doesn't care about operand types
+  // leaving only registry class so SSrc_32 operand turns into SReg_32
+  // and therefore we accept immediates and literals here as well
+  return decodeSrcOp(OPW32, Val);
+}
+
 MCOperand AMDGPUDisassembler::decodeOperand_SReg_64(unsigned Val) const {
   return decodeSrcOp(OPW64, Val);
 }
 
 MCOperand AMDGPUDisassembler::decodeOperand_SReg_64_XEXEC(unsigned Val) const {
   return decodeSrcOp(OPW64, Val);
 }
 

lib/Target/AMDGPU/SIRegisterInfo.td

 }
 
 // Register class for all scalar registers (SGPRs + Special Registers)
 def SReg_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
   (add SReg_32_XM0, M0_CLASS, EXEC_LO, EXEC_HI, SReg_32_XEXEC_HI)> {
   let AllocationPriority = 7;
 }
 
+def SRegOrLds_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
+  (add SReg_32_XM0, M0_CLASS, EXEC_LO, EXEC_HI, SReg_32_XEXEC_HI, LDS_DIRECT_CLASS)> {
+  let isAllocatable = 0;
+}
+
 def SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, v4i16, v4f16], 32, (add SGPR_64Regs)> {
   let CopyCost = 1;
   let AllocationPriority = 8;
 }
 
 def TTMP_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64, v4i16, v4f16], 32, (add TTMP_64Regs)> {
   let isAllocatable = 0;
 }

 
 def SReg_512 : RegisterClass<"AMDGPU", [v16i32, v16f32], 32,
   (add SGPR_512, TTMP_512)> {
   // Requires 8 s_mov_b64 to copy
   let CopyCost = 8;
   let AllocationPriority = 12;
 }
 
+def VRegOrLds_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
+                                 (add VGPR_32, LDS_DIRECT_CLASS)> {
+  let isAllocatable = 0;
+}
+
 // Register class for all vector registers (VGPRs + Interploation Registers)
 def VReg_64 : RegisterClass<"AMDGPU", [i64, f64, v2i32, v2f32, v4f16, v4i16], 32, (add VGPR_64)> {
   let Size = 64;
 
   // Requires 2 v_mov_b32 to copy
   let CopyCost = 2;
   let AllocationPriority = 2;
 }
multiclass RegInlineOperand <string rc, string MatchName>
   : SIRegOperand<rc, MatchName, "OPERAND_REG_INLINE_C">;
 
 //===----------------------------------------------------------------------===//
 //  SSrc_* Operands with an SGPR or a 32-bit immediate
 //===----------------------------------------------------------------------===//
 
 defm SSrc : RegImmOperand<"SReg", "SSrc">;
 
+def SSrcOrLds_b32 : RegisterOperand<SRegOrLds_32> {
+  let OperandNamespace = "AMDGPU";
+  let OperandType = "OPERAND_REG_IMM_INT32";
+  let ParserMatchClass = RegImmMatcher<"SSrcOrLdsB32">;
+}
+
 //===----------------------------------------------------------------------===//
 //  SCSrc_* Operands with an SGPR or a inline constant
 //===----------------------------------------------------------------------===//
 
 defm SCSrc : RegInlineOperand<"SReg", "SCSrc"> ;
 
+def SCSrcOrLds_b32 : RegisterOperand<SRegOrLds_32> {
+  let OperandNamespace = "AMDGPU";
+  let OperandType = "OPERAND_REG_INLINE_C_INT32";
+  let ParserMatchClass = RegImmMatcher<"SCSrcOrLdsB32">;
+}
+
 def SCSrc_i1 : RegisterOperand<SReg_64_XEXEC>;
 
 //===----------------------------------------------------------------------===//
 //  VSrc_* Operands with an SGPR, VGPR or a 32-bit immediate
 //===----------------------------------------------------------------------===//
 
 defm VSrc : RegImmOperand<"VS", "VSrc">;
 

lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

 }
 
 // Avoid using MCRegisterClass::getSize, since that function will go away
 // (move from MC* level to Target* level). Return size in bits.
 unsigned getRegBitWidth(unsigned RCID) {
   switch (RCID) {
   case AMDGPU::SGPR_32RegClassID:
   case AMDGPU::VGPR_32RegClassID:
+  case AMDGPU::VRegOrLds_32RegClassID:
   case AMDGPU::VS_32RegClassID:
   case AMDGPU::SReg_32RegClassID:
   case AMDGPU::SReg_32_XM0RegClassID:
+  case AMDGPU::SRegOrLds_32RegClassID:
     return 32;
   case AMDGPU::SGPR_64RegClassID:
   case AMDGPU::VS_64RegClassID:
   case AMDGPU::SReg_64RegClassID:
   case AMDGPU::VReg_64RegClassID:
   case AMDGPU::SReg_64_XEXECRegClassID:
     return 64;
   case AMDGPU::VReg_96RegClassID:

lib/Target/AMDGPU/VOP1Instructions.td

 let isMoveImm = 1, isReMaterializable = 1, isAsCheapAsAMove = 1 in {
 defm V_MOV_B32 : VOP1Inst <"v_mov_b32", VOP_I32_I32>;
 } // End isMoveImm = 1
 
 // FIXME: Specify SchedRW for READFIRSTLANE_B32
 // TODO: Make profile for this, there is VOP3 encoding also
 def V_READFIRSTLANE_B32 :
   InstSI <(outs SReg_32:$vdst),
-    (ins VGPR_32:$src0),
+    (ins VRegOrLds_32:$src0),
     "v_readfirstlane_b32 $vdst, $src0",
     [(set i32:$vdst, (int_amdgcn_readfirstlane i32:$src0))]>,
   Enc32 {
 
   let isCodeGenOnly = 0;
   let UseNamedOperandTable = 1;
 
   let Size = 4;

lib/Target/AMDGPU/VOP2Instructions.td

def VOP2e_I32_I32_I32_I1 : VOPProfile<[i32, i32, i32, i1]> {
   let HasExtDPP = 1;
   let HasExtSDWA = 1;
   let HasExtSDWA9 = 1;
 }
 
 def VOP_READLANE : VOPProfile<[i32, i32, i32]> {
   let Outs32 = (outs SReg_32:$vdst);
   let Outs64 = Outs32;
-  let Ins32 = (ins VGPR_32:$src0, SCSrc_b32:$src1);
+  let Ins32 = (ins VRegOrLds_32:$src0, SCSrc_b32:$src1);
   let Ins64 = Ins32;
   let Asm32 = " $vdst, $src0, $src1";
   let Asm64 = Asm32;
 
   let HasExt = 0;
   let HasExtDPP = 0;
   let HasExtSDWA = 0;
   let HasExtSDWA9 = 0;
 }
 
 def VOP_WRITELANE : VOPProfile<[i32, i32, i32, i32]> {
   let Outs32 = (outs VGPR_32:$vdst);
   let Outs64 = Outs32;
-  let Ins32 = (ins SCSrc_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in);
+  let Ins32 = (ins SCSrcOrLds_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in);
   let Ins64 = Ins32;
   let Asm32 = " $vdst, $src0, $src1";
   let Asm64 = Asm32;
   let HasSrc2 = 0;
   let HasSrc2Mods = 0;
 
   let HasExt = 0;
   let HasExtDPP = 0;

 defm V_SUB_I32            : VOP2be_Real_e32e64_si <0x26>;
 defm V_SUBREV_I32         : VOP2be_Real_e32e64_si <0x27>;
 defm V_ADDC_U32           : VOP2be_Real_e32e64_si <0x28>;
 defm V_SUBB_U32           : VOP2be_Real_e32e64_si <0x29>;
 defm V_SUBBREV_U32        : VOP2be_Real_e32e64_si <0x2a>;
 
 defm V_READLANE_B32       : VOP2_Real_si <0x01>;
 
-let InOperandList = (ins SSrc_b32:$src0, SCSrc_b32:$src1, VSrc_b32:$vdst_in) in {
+let InOperandList = (ins SSrcOrLds_b32:$src0, SCSrc_b32:$src1, VSrc_b32:$vdst_in) in {
 defm V_WRITELANE_B32      : VOP2_Real_si <0x02>;
 }
 
 defm V_MAC_LEGACY_F32     : VOP2_Real_e32e64_si <0x6>;
 defm V_MIN_LEGACY_F32     : VOP2_Real_e32e64_si <0xd>;
 defm V_MAX_LEGACY_F32     : VOP2_Real_e32e64_si <0xe>;
 defm V_LSHR_B32           : VOP2_Real_e32e64_si <0x15>;
 defm V_ASHR_I32           : VOP2_Real_e32e64_si <0x17>;

lib/Target/AMDGPU/VOP3Instructions.td

 let isAsmParserOnly = 1 in {
 defm V_MUL_LO_I32       : VOP3_Real_vi <0x285>;
 }
 
 defm V_MUL_HI_U32       : VOP3_Real_vi <0x286>;
 defm V_MUL_HI_I32       : VOP3_Real_vi <0x287>;
 
 defm V_READLANE_B32     : VOP3_Real_vi <0x289>;
+
+let InOperandList = (ins SCSrc_b32:$src0, SCSrc_b32:$src1, VGPR_32:$vdst_in) in {
 defm V_WRITELANE_B32    : VOP3_Real_vi <0x28a>;
+}
 
 defm V_LSHLREV_B64      : VOP3_Real_vi <0x28f>;
 defm V_LSHRREV_B64      : VOP3_Real_vi <0x290>;
 defm V_ASHRREV_I64      : VOP3_Real_vi <0x291>;
 defm V_TRIG_PREOP_F64   : VOP3_Real_vi <0x292>;
 
 defm V_LSHL_ADD_U32 : VOP3_Real_vi <0x1fd>;
 defm V_ADD_LSHL_U32 : VOP3_Real_vi <0x1fe>;

test/MC/AMDGPU/lds_direct-ci.s

+// RUN: llvm-mc -arch=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck %s --check-prefix=CI
+
+v_readfirstlane_b32 s0, lds_direct
+// CI: v_readfirstlane_b32 s0, src_lds_direct ; encoding: [0xfe,0x04,0x00,0x7e]
+
+v_readlane_b32 s0, lds_direct, s0
+// CI: v_readlane_b32 s0, src_lds_direct, s0 ; encoding: [0xfe,0x00,0x00,0x02]
+
+v_writelane_b32 v0, lds_direct, s0
+// CI: v_writelane_b32 v0, src_lds_direct, s0 ; encoding: [0xfe,0x00,0x00,0x04]

test/MC/AMDGPU/lds_direct-err.s

 // RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 %s 2>&1 | FileCheck %s --check-prefix=NOGFX9
 
 //---------------------------------------------------------------------------//
 // lds_direct may be used only with vector ALU instructions
 //---------------------------------------------------------------------------//
 
 s_and_b32 s2, lds_direct, s1
-// NOGFX9: error
+// NOGFX9: error: invalid operand for instruction
 
 //---------------------------------------------------------------------------//
-// lds_direct may not be used with V_{LSHL,LSHR,ASHL}REV opcodes
+// lds_direct may not be used with "REV" opcodes
 //---------------------------------------------------------------------------//
 
 v_ashrrev_i16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_ashrrev_i32 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_lshlrev_b16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_lshlrev_b32 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_lshrrev_b16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_lshrrev_b32 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_pk_ashrrev_i16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_pk_lshlrev_b16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_pk_lshrrev_b16 v0, lds_direct, v0
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
+
+v_subbrev_co_u32 v0, vcc, src_lds_direct, v0, vcc
+// NOGFX9: error: invalid use of lds_direct
+
+v_subrev_co_u32 v0, vcc, src_lds_direct, v0
+// NOGFX9: error: invalid use of lds_direct
+
+v_subrev_f16 v0, src_lds_direct, v0
+// NOGFX9: error: invalid use of lds_direct
+
+v_subrev_u16 v0, src_lds_direct, v0
+// NOGFX9: error: invalid use of lds_direct
+
+v_subrev_u32 v0, src_lds_direct, v0
+// NOGFX9: error: invalid use of lds_direct
+
+//---------------------------------------------------------------------------//
+// lds_direct may not be used with v_writelane_b32 for VI/GFX9
+//---------------------------------------------------------------------------//
+
+v_writelane_b32 v0, lds_direct, s0
+// NOGFX9: error: instruction not supported on this GPU
 
 //---------------------------------------------------------------------------//
 // lds_direct cannot be used with 64-bit and larger operands
 //---------------------------------------------------------------------------//
 
 v_add_f64 v[0:1], lds_direct, v[0:1]
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 //---------------------------------------------------------------------------//
 // Only SRC0 may specify lds_direct
 //---------------------------------------------------------------------------//
 
 v_add_i32 v0, v0, lds_direct
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct
 
 v_add_i32 lds_direct, v0, v0
-// NOGFX9: error
+// NOGFX9: error: invalid operand for instruction
 
 v_fma_f32 v0, v0, v0, lds_direct
-// NOGFX9: error
+// NOGFX9: error: invalid use of lds_direct

test/MC/AMDGPU/lds_direct.s

 // GFX9: v_fract_f32_e32 v0, src_lds_direct ; encoding: [0xfe,0x36,0x00,0x7e]
 
 v_fract_f32_e64 v0, src_lds_direct
 // GFX9: v_fract_f32_e64 v0, src_lds_direct ; encoding: [0x00,0x00,0x5b,0xd1,0xfe,0x00,0x00,0x00]
 
 v_cvt_f16_u16 v0, src_lds_direct
 // GFX9: v_cvt_f16_u16_e32 v0, src_lds_direct ; encoding: [0xfe,0x72,0x00,0x7e]
 
+v_readfirstlane_b32 s0, src_lds_direct
+// GFX9: v_readfirstlane_b32 s0, src_lds_direct ; encoding: [0xfe,0x04,0x00,0x7e]
+
 //---------------------------------------------------------------------------//
 // VOP2/3
 //---------------------------------------------------------------------------//
 
 v_cndmask_b32 v0, src_lds_direct, v0, vcc
 // GFX9: v_cndmask_b32_e32 v0, src_lds_direct, v0, vcc ; encoding: [0xfe,0x00,0x00,0x00]
 
 v_cndmask_b32_e64 v0, src_lds_direct, v0, s[0:1]

 // GFX9: v_fma_f32 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xcb,0xd1,0xfe,0x00,0x02,0x04]
 
 v_min3_i16 v0, src_lds_direct, v0, v0
 // GFX9: v_min3_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xf5,0xd1,0xfe,0x00,0x02,0x04]
 
 v_max3_f16 v0, src_lds_direct, v0, v0
 // GFX9: v_max3_f16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xf7,0xd1,0xfe,0x00,0x02,0x04]
 
+v_readlane_b32 s0, src_lds_direct, s0
+// GFX9: v_readlane_b32 s0, src_lds_direct, s0 ; encoding: [0x00,0x00,0x89,0xd2,0xfe,0x00,0x00,0x00]
+
 //---------------------------------------------------------------------------//
 // VOP3P
 //---------------------------------------------------------------------------//
 
 v_pk_mad_i16 v0, src_lds_direct, v0, v0
 // GFX9: v_pk_mad_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c]
 
 v_pk_add_i16 v0, src_lds_direct, v0

 // GFX9: v_cmpx_neq_f32_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0xba,0x7c]
 
 //---------------------------------------------------------------------------//
 // lds_direct alias
 //---------------------------------------------------------------------------//
 
 v_cmp_lt_f16 vcc, lds_direct, v0
 // GFX9: v_cmp_lt_f16_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x42,0x7c]
-
-//---------------------------------------------------------------------------//
-// FIXME: enable lds_direct for the following opcodes and add tests
-//---------------------------------------------------------------------------//
-
-//v_readfirstlane_b32 s0, src_lds_direct
-//v_readlane_b32 s0, src_lds_direct, s0

test/MC/Disassembler/AMDGPU/lds_direct_gfx9.txt

 # GFX9: v_pk_mad_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c]
 0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c
 
 # GFX9: v_pk_mul_lo_u16 v0, src_lds_direct, v0 ; encoding: [0x00,0x00,0x81,0xd3,0xfe,0x00,0x02,0x18]
 0x00,0x00,0x81,0xd3,0xfe,0x00,0x02,0x18
 
 # GFX9: v_cmpx_le_i32_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0xa6,0x7d]
 0xfe,0x00,0xa6,0x7d
+
+# GFX9: v_readlane_b32 s0, src_lds_direct, s0 ; encoding: [0x00,0x00,0x89,0xd2,0xfe,0x00,0x00,0x00]
+0x00,0x00,0x89,0xd2,0xfe,0x00,0x00,0x00
+
+# GFX9: v_readfirstlane_b32 s0, src_lds_direct ; encoding: [0xfe,0x04,0x00,0x7e]
+0xfe,0x04,0x00,0x7e