[AMDGPU][MC][GFX8][GFX9] Added XNACK_MASK support

[dp]

Summary

See bug 35764: https://bugs.llvm.org/show_bug.cgi?id=35764

[dp]

Removed incorrect assert in disassembler;
added a few tests for GFX9

[arsenm]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

Shouldn't this be xnack enabled (or at least >= VI)

[dp]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

Thanks, Matt! I'll correct that shortly.

[artem.tamazov]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

I am not sure. https://github.com/ROCm-Developer-Tools/ROCm-ComputeABI-Doc/blob/master/AMDGPU-ABI.md#instruction-set-architecture says:
AMD AMDGPU 8 0 1 GFX8, XNACK enabled A10-8700 series APU

[arsenm]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

I suppose for encoding/decoding purposes, I suppose it won't make a difference, it will just produce an unusable program for the target. Maybe a warning?

[dp]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

We have a FeatureXNACK, but it is enabled only for 8_0_1 and 8_1_0. And not for any of GFX9 targets. So I do not think we could use that. Is the feature list for GFX9 incomplete?

[dp]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

Do you know which GFX9 targets support this feature?

[kzhuravl]

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

2635

+ @t-tye

I think this should check the FeatureXNACK (and maybe >=VI, xnack started from VI). Users can pass -mxnack, -mno-xnack regardless the target from clang, which will set FeatureXNACK to true or false respectively. Runtime has to ensure it is valid.

https://llvm.org/docs/AMDGPUUsage.html#target-features

[dp]

Corrected XNACK availability check as suggested by Matt and Konstantin; updated tests.

[arsenm]

LGTM

lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp

for (const MachineInstr &MI : MBB) {
           Info.UsesVCC = true;
           continue;
 
         case AMDGPU::FLAT_SCR:
         case AMDGPU::FLAT_SCR_LO:
         case AMDGPU::FLAT_SCR_HI:
           continue;
 
+        case AMDGPU::XNACK_MASK:
+        case AMDGPU::XNACK_MASK_LO:
+        case AMDGPU::XNACK_MASK_HI:
+          llvm_unreachable("xnack_mask registers should not be used");
+
         case AMDGPU::TBA:
         case AMDGPU::TBA_LO:
         case AMDGPU::TBA_HI:
         case AMDGPU::TMA:
         case AMDGPU::TMA_LO:
         case AMDGPU::TMA_HI:
           llvm_unreachable("trap handler registers should not be used");
 

lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
       Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
       Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
       Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
     }
     KernelScope.initialize(getContext());
   }
 
+  bool hasXNACK() const {
+    return AMDGPU::hasXNACK(getSTI());
+  }
+
   bool isSI() const {
     return AMDGPU::isSI(getSTI());
   }
 
   bool isCI() const {
     return AMDGPU::isCI(getSTI());
   }
 
static int getRegClass(RegisterKind Is, unsigned RegWidth) {
   return -1;
 }
 
 static unsigned getSpecialRegForName(StringRef RegName) {
   return StringSwitch<unsigned>(RegName)
     .Case("exec", AMDGPU::EXEC)
     .Case("vcc", AMDGPU::VCC)
     .Case("flat_scratch", AMDGPU::FLAT_SCR)
+    .Case("xnack_mask", AMDGPU::XNACK_MASK)
     .Case("m0", AMDGPU::M0)
     .Case("scc", AMDGPU::SCC)
     .Case("tba", AMDGPU::TBA)
     .Case("tma", AMDGPU::TMA)
     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
+    .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
+    .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
     .Case("vcc_lo", AMDGPU::VCC_LO)
     .Case("vcc_hi", AMDGPU::VCC_HI)
     .Case("exec_lo", AMDGPU::EXEC_LO)
     .Case("exec_hi", AMDGPU::EXEC_HI)
     .Case("tma_lo", AMDGPU::TMA_LO)
     .Case("tma_hi", AMDGPU::TMA_HI)
     .Case("tba_lo", AMDGPU::TBA_LO)
     .Case("tba_hi", AMDGPU::TBA_HI)
if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
       RegWidth = 2;
       return true;
     }
     if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
       Reg = AMDGPU::FLAT_SCR;
       RegWidth = 2;
       return true;
     }
+    if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
+      Reg = AMDGPU::XNACK_MASK;
+      RegWidth = 2;
+      return true;
+    }
     if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
       Reg = AMDGPU::VCC;
       RegWidth = 2;
       return true;
     }
     if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
       Reg = AMDGPU::TBA;
       RegWidth = 2;
bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
   switch (RegNo) {
   case AMDGPU::TBA:
   case AMDGPU::TBA_LO:
   case AMDGPU::TBA_HI:
   case AMDGPU::TMA:
   case AMDGPU::TMA_LO:
   case AMDGPU::TMA_HI:
     return !isGFX9();
+  case AMDGPU::XNACK_MASK:
+  case AMDGPU::XNACK_MASK_LO:
+  case AMDGPU::XNACK_MASK_HI:
+    return !isCI() && !isSI() && hasXNACK();
   default:
     break;
   }
 
   if (isCI())
     return true;
 
   if (isSI()) {

lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp

 }
 
 MCOperand AMDGPUDisassembler::decodeSpecialReg32(unsigned Val) const {
   using namespace AMDGPU;
 
   switch (Val) {
   case 102: return createRegOperand(FLAT_SCR_LO);
   case 103: return createRegOperand(FLAT_SCR_HI);
-    // ToDo: no support for xnack_mask_lo/_hi register
-  case 104:
-  case 105: break;
+  case 104: return createRegOperand(XNACK_MASK_LO);
+  case 105: return createRegOperand(XNACK_MASK_HI);
   case 106: return createRegOperand(VCC_LO);
   case 107: return createRegOperand(VCC_HI);
   case 108: assert(!isGFX9()); return createRegOperand(TBA_LO);
   case 109: assert(!isGFX9()); return createRegOperand(TBA_HI);
   case 110: assert(!isGFX9()); return createRegOperand(TMA_LO);
   case 111: assert(!isGFX9()); return createRegOperand(TMA_HI);
   case 124: return createRegOperand(M0);
   case 126: return createRegOperand(EXEC_LO);
MCOperand AMDGPUDisassembler::decodeSpecialReg32(unsigned Val) const {
   return errOperand(Val, "unknown operand encoding " + Twine(Val));
 }
 
 MCOperand AMDGPUDisassembler::decodeSpecialReg64(unsigned Val) const {
   using namespace AMDGPU;
 
   switch (Val) {
   case 102: return createRegOperand(FLAT_SCR);
+  case 104: return createRegOperand(XNACK_MASK);
   case 106: return createRegOperand(VCC);
   case 108: assert(!isGFX9()); return createRegOperand(TBA);
   case 110: assert(!isGFX9()); return createRegOperand(TMA);
   case 126: return createRegOperand(EXEC);
   default: break;
   }
   return errOperand(Val, "unknown operand encoding " + Twine(Val));
 }

lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp

case AMDGPU::EXEC:
     O << "exec";
     return;
   case AMDGPU::M0:
     O << "m0";
     return;
   case AMDGPU::FLAT_SCR:
     O << "flat_scratch";
     return;
+  case AMDGPU::XNACK_MASK:
+    O << "xnack_mask";
+    return;
   case AMDGPU::VCC_LO:
     O << "vcc_lo";
     return;
   case AMDGPU::VCC_HI:
     O << "vcc_hi";
     return;
   case AMDGPU::TBA_LO:
     O << "tba_lo";
case AMDGPU::EXEC_HI:
     O << "exec_hi";
     return;
   case AMDGPU::FLAT_SCR_LO:
     O << "flat_scratch_lo";
     return;
   case AMDGPU::FLAT_SCR_HI:
     O << "flat_scratch_hi";
     return;
+  case AMDGPU::XNACK_MASK_LO:
+    O << "xnack_mask_lo";
+    return;
+  case AMDGPU::XNACK_MASK_HI:
+    O << "xnack_mask_hi";
+    return;
   case AMDGPU::FP_REG:
   case AMDGPU::SP_REG:
   case AMDGPU::SCRATCH_WAVE_OFFSET_REG:
   case AMDGPU::PRIVATE_RSRC_REG:
     llvm_unreachable("pseudo-register should not ever be emitted");
   default:
     break;
   }

lib/Target/AMDGPU/SIRegisterInfo.cpp

BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   reserveRegisterTuples(Reserved, AMDGPU::M0);
 
   // Reserve the memory aperture registers.
   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_BASE);
   reserveRegisterTuples(Reserved, AMDGPU::SRC_SHARED_LIMIT);
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
 
+  // Reserve xnack_mask registers - support is not implemented in Codegen.
+  reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK);
+
   // Reserve Trap Handler registers - support is not implemented in Codegen.
   reserveRegisterTuples(Reserved, AMDGPU::TBA);
   reserveRegisterTuples(Reserved, AMDGPU::TMA);
   reserveRegisterTuples(Reserved, AMDGPU::TTMP0_TTMP1);
   reserveRegisterTuples(Reserved, AMDGPU::TTMP2_TTMP3);
   reserveRegisterTuples(Reserved, AMDGPU::TTMP4_TTMP5);
   reserveRegisterTuples(Reserved, AMDGPU::TTMP6_TTMP7);
   reserveRegisterTuples(Reserved, AMDGPU::TTMP8_TTMP9);

lib/Target/AMDGPU/SIRegisterInfo.td

 def SCC : SIReg<"scc", 253>;
 def M0 : SIReg <"m0", 124>;
 
 def SRC_SHARED_BASE : SIReg<"src_shared_base", 235>;
 def SRC_SHARED_LIMIT : SIReg<"src_shared_limit", 236>;
 def SRC_PRIVATE_BASE : SIReg<"src_private_base", 237>;
 def SRC_PRIVATE_LIMIT : SIReg<"src_private_limit", 238>;
 
+def XNACK_MASK_LO : SIReg<"xnack_mask_lo", 104>;
+def XNACK_MASK_HI : SIReg<"xnack_mask_hi", 105>;
+
+def XNACK_MASK : RegisterWithSubRegs<"xnack_mask", [XNACK_MASK_LO, XNACK_MASK_HI]>,
+                 DwarfRegAlias<XNACK_MASK_LO> {
+  let Namespace = "AMDGPU";
+  let SubRegIndices = [sub0, sub1];
+  let HWEncoding = 104;
+}
+
 // Trap handler registers
 def TBA_LO : SIReg<"tba_lo", 108>;
 def TBA_HI : SIReg<"tba_hi", 109>;
 
 def TBA : RegisterWithSubRegs<"tba", [TBA_LO, TBA_HI]>,
           DwarfRegAlias<TBA_LO> {
   let Namespace = "AMDGPU";
   let SubRegIndices = [sub0, sub1];
def Pseudo_SReg_128 : RegisterClass<"AMDGPU", [v4i32, v2i64], 32,
   (add PRIVATE_RSRC_REG)> {
   let isAllocatable = 0;
   let CopyCost = -1;
 }
 
 // Subset of SReg_32 without M0 for SMRD instructions and alike.
 // See comments in SIInstructions.td for more info.
 def SReg_32_XM0_XEXEC : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
-  (add SGPR_32, VCC_LO, VCC_HI, FLAT_SCR_LO, FLAT_SCR_HI,
+  (add SGPR_32, VCC_LO, VCC_HI, FLAT_SCR_LO, FLAT_SCR_HI, XNACK_MASK_LO, XNACK_MASK_HI,
    TTMP_32, TMA_LO, TMA_HI, TBA_LO, TBA_HI, SRC_SHARED_BASE, SRC_SHARED_LIMIT,
    SRC_PRIVATE_BASE, SRC_PRIVATE_LIMIT)> {
   let AllocationPriority = 7;
 }
 
 def SReg_32_XEXEC_HI : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
   (add SReg_32_XM0_XEXEC, EXEC_LO, M0_CLASS)> {
   let AllocationPriority = 7;
def SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64], 32, (add SGPR_64Regs)> {
   let AllocationPriority = 8;
 }
 
 def TTMP_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64], 32, (add TTMP_64Regs)> {
   let isAllocatable = 0;
 }
 
 def SReg_64_XEXEC : RegisterClass<"AMDGPU", [v2i32, i64, f64, i1], 32,
-  (add SGPR_64, VCC, FLAT_SCR, TTMP_64, TBA, TMA)> {
+  (add SGPR_64, VCC, FLAT_SCR, XNACK_MASK, TTMP_64, TBA, TMA)> {
   let CopyCost = 1;
   let AllocationPriority = 8;
 }
 
 def SReg_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64, i1], 32,
   (add SReg_64_XEXEC, EXEC)> {
   let CopyCost = 1;
   let AllocationPriority = 8;

lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h

inline bool isKernel(CallingConv::ID CC) {
   case CallingConv::AMDGPU_KERNEL:
   case CallingConv::SPIR_KERNEL:
     return true;
   default:
     return false;
   }
 }
 
+bool hasXNACK(const MCSubtargetInfo &STI);
+
 bool isSI(const MCSubtargetInfo &STI);
 bool isCI(const MCSubtargetInfo &STI);
 bool isVI(const MCSubtargetInfo &STI);
 bool isGFX9(const MCSubtargetInfo &STI);
 
 /// \brief Is Reg - scalar register
 bool isSGPR(unsigned Reg, const MCRegisterInfo* TRI);
 

lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp

bool isEntryFunctionCC(CallingConv::ID CC) {
   case CallingConv::AMDGPU_HS:
   case CallingConv::AMDGPU_LS:
     return true;
   default:
     return false;
   }
 }
 
+bool hasXNACK(const MCSubtargetInfo &STI) {
+  return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
+}
+
 bool isSI(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureSouthernIslands];
 }
 
 bool isCI(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureSeaIslands];
 }
 

test/MC/AMDGPU/xnack-mask.s

+// RUN: not llvm-mc -arch=amdgcn -mcpu=tahiti -show-encoding %s 2>&1 | FileCheck -check-prefix=NOSICIVI %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=hawaii -show-encoding %s 2>&1 | FileCheck -check-prefix=NOSICIVI %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s 2>&1 | FileCheck -check-prefix=NOSICIVI %s
+
+// RUN: not llvm-mc -arch=amdgcn -mcpu=stoney -show-encoding %s 2>&1 | FileCheck -check-prefix=XNACKERR %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=stoney -show-encoding %s | FileCheck -check-prefix=XNACK %s
+
+s_mov_b64 xnack_mask, -1
+// NOSICIVI: error: not a valid operand.
+// XNACK:    s_mov_b64 xnack_mask, -1 ; encoding: [0xc1,0x01,0xe8,0xbe]
+
+s_mov_b32 xnack_mask_lo, -1
+// NOSICIVI: error: not a valid operand.
+// XNACK:    s_mov_b32 xnack_mask_lo, -1 ; encoding: [0xc1,0x00,0xe8,0xbe]
+
+s_mov_b32 xnack_mask_hi, -1
+// NOSICIVI: error: not a valid operand.
+// XNACK:    s_mov_b32 xnack_mask_hi, -1 ; encoding: [0xc1,0x00,0xe9,0xbe]
+
+s_mov_b32 xnack_mask, -1
+// NOSICIVI: error: not a valid operand.
+// XNACKERR: error: invalid operand for instruction
+
+s_mov_b64 xnack_mask_lo, -1
+// NOSICIVI: error: not a valid operand.
+// XNACKERR: error: invalid operand for instruction
+
+s_mov_b64 xnack_mask_hi, -1
+// NOSICIVI: error: not a valid operand.
+// XNACKERR: error: invalid operand for instruction

test/MC/Disassembler/AMDGPU/sop1_vi.txt

 0xff 0x00 0x81 0xbe 0x64 0x00 0x00 0x00
 
 # VI:   s_mov_b32 s1, 0x80000000 ; encoding: [0xff,0x00,0x81,0xbe,0x00,0x00,0x00,0x80]
 0xff 0x00 0x81 0xbe 0x00 0x00 0x00 0x80
 
 # VI:   s_mov_b32 s0, 0xfe5163ab ; encoding: [0xff,0x00,0x80,0xbe,0xab,0x63,0x51,0xfe]
 0xff 0x00 0x80 0xbe 0xab 0x63 0x51 0xfe
 
+# VI:   s_mov_b32 xnack_mask_lo, -1 ; encoding: [0xc1,0x00,0xe8,0xbe]
+0xc1,0x00,0xe8,0xbe
+
+# VI:   s_mov_b32 xnack_mask_hi, -1 ; encoding: [0xc1,0x00,0xe9,0xbe]
+0xc1,0x00,0xe9,0xbe
+
 # VI:   s_mov_b64 s[2:3], s[4:5] ; encoding: [0x04,0x01,0x82,0xbe]
 0x04 0x01 0x82 0xbe
 
-# FIXME:   s_mov_b64 s[2:3], -1 ; encoding: [0xc1,0x01,0x82,0xbe]
+# VI:   s_mov_b64 s[2:3], -1 ; encoding: [0xc1,0x01,0x82,0xbe]
 0xc1 0x01 0x82 0xbe
 
+# VI: s_mov_b64 xnack_mask, -1 ; encoding: [0xc1,0x01,0xe8,0xbe]
+0xc1,0x01,0xe8,0xbe
+
 # VI:   s_mov_b64 s[2:3], 0xffffffff ; encoding: [0xff,0x01,0x82,0xbe,0xff,0xff,0xff,0xff]
 0xff 0x01 0x82 0xbe 0xff 0xff 0xff 0xff
 
 # VI:   s_mov_b64 s[0:1], 0x80000000 ; encoding: [0xff,0x01,0x80,0xbe,0x00,0x00,0x00,0x80]
 0xff 0x01 0x80 0xbe 0x00 0x00 0x00 0x80
 
 # VI:   s_cmov_b32 s1, 0xc8 ; encoding: [0xff,0x02,0x81,0xbe,0xc8,0x00,0x00,0x00]
 0xff 0x02 0x81 0xbe 0xc8 0x00 0x00 0x00

test/MC/Disassembler/AMDGPU/vop3_gfx9.txt

 0x05,0x00,0xa0,0xd3,0x65,0x04,0x0e,0x04
 
 # GFX9: v_mad_mix_f32 v5, flat_scratch_lo, v2, v3    ; encoding: [0x05,0x00,0xa0,0xd3,0x66,0x04,0x0e,0x04]
 0x05,0x00,0xa0,0xd3,0x66,0x04,0x0e,0x04
 
 # GFX9: v_mad_mix_f32 v5, flat_scratch_hi, v2, v3    ; encoding: [0x05,0x00,0xa0,0xd3,0x67,0x04,0x0e,0x04]
 0x05,0x00,0xa0,0xd3,0x67,0x04,0x0e,0x04
 
+# GFX9: v_mad_mix_f32 v5, xnack_mask_hi, v2, v3 ; encoding: [0x05,0x00,0xa0,0xd3,0x69,0x04,0x0e,0x04]
+0x05,0x00,0xa0,0xd3,0x69,0x04,0x0e,0x04
+
 # GFX9: v_mad_mix_f32 v5, vcc_lo, v2, v3    ; encoding: [0x05,0x00,0xa0,0xd3,0x6a,0x04,0x0e,0x04]
 0x05,0x00,0xa0,0xd3,0x6a,0x04,0x0e,0x04
 
 # GFX9: v_mad_mix_f32 v5, vcc_hi, v2, v3    ; encoding: [0x05,0x00,0xa0,0xd3,0x6b,0x04,0x0e,0x04]
 0x05,0x00,0xa0,0xd3,0x6b,0x04,0x0e,0x04
 
 # GFX9: v_mad_mix_f32 v5, m0, v2, v3    ; encoding: [0x05,0x00,0xa0,0xd3,0x7c,0x04,0x0e,0x04]
 0x05,0x00,0xa0,0xd3,0x7c,0x04,0x0e,0x04

 # GFX9: v_interp_p2_legacy_f16 v5, v2, attr0.w, v3 ; encoding: [0x05,0x00,0x76,0xd2,0xc0,0x04,0x0e,0x04]
 0x05,0x00,0x76,0xd2,0xc0,0x04,0x0e,0x04
 
 # GFX9: v_interp_p2_legacy_f16 v5, v2, attr0.x, v3 high ; encoding: [0x05,0x00,0x76,0xd2,0x00,0x05,0x0e,0x04]
 0x05,0x00,0x76,0xd2,0x00,0x05,0x0e,0x04
 
 # GFX9: v_interp_p2_legacy_f16 v5, v2, attr0.x, v3 clamp ; encoding: [0x05,0x80,0x76,0xd2,0x00,0x04,0x0e,0x04]
 0x05,0x80,0x76,0xd2,0x00,0x04,0x0e,0x04
+
+# GFX9: v_add_f64 v[5:6], xnack_mask, v[2:3] ; encoding: [0x05,0x00,0x80,0xd2,0x68,0x04,0x02,0x00]
+0x05,0x00,0x80,0xd2,0x68,0x04,0x02,0x00