[AMDGPU][True16] Support emitting copies between different register sizes.

[arsenm]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

741–745

Is this actually reachable? I forget how exactly we ended up with this partial 16-bit register thing

[rampitec]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

742

Are these tabs?

[kosarev]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

741–745

I'm going to try to understand more about what's going on here, but maybe @Joe_Nash already knows the answer as he was working on that bit.

742

Nope, just re-indendting with proper spaces.

[Joe_Nash]

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

741–745

It is covered by lo16-32bit-physreg-copy.mir.

In theory it is reachable on any target with 16 bit instructions (excluding GF11 with True 16 bit instructions). This code converts
COPY %1:vgpr_32 = %2:vgpr_lo16 or COPY %1:vgpr_lo16 = %2:vgpr_32
into v_mov_b32 on those targets.
On GFX11 it converts to v_mov_b16

Now as to whether the code sequence in that mir test (or any use of a VGPR_LO16 register) is produced in any legitimate shader, I don't think so. That probably needs to be verified empirically on a test corpus, but then this code and I think VGPR_LO16/ VGPR_HI16 can be removed.

[foad]

Support generating differently-sized register transfers.

They are called copies, not transfers. How about: "Support emitting copies between different register sizes"?

[kosarev]

Change the commit title as suggested and remove the unused code handling
the non-True16 case along with the test covering it.

[Joe_Nash]

In D156105#4550060, @kosarev wrote:

Change the commit title as suggested and remove the unused code handling
the non-True16 case along with the test covering it.

Can you please do this case removal as a separate patch?

[kosarev]

Restore the non-True16 case branch to then remove it with a separate patch.

[kosarev]

Ping.

[Joe_Nash]

LGTM

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

if (KillSrc)
     LastMI->addRegisterKilled(SrcReg, &RI);
 }
 
 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MI,
                               const DebugLoc &DL, MCRegister DestReg,
                               MCRegister SrcReg, bool KillSrc) const {
   const TargetRegisterClass *RC = RI.getPhysRegBaseClass(DestReg);
+  unsigned Size = RI.getRegSizeInBits(*RC);
+  const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);
+  unsigned SrcSize = RI.getRegSizeInBits(*SrcRC);
 
-  // FIXME: This is hack to resolve copies between 16 bit and 32 bit
-  // registers until all patterns are fixed.
-  if (Fix16BitCopies &&
-      ((RI.getRegSizeInBits(*RC) == 16) ^
-       (RI.getRegSizeInBits(*RI.getPhysRegBaseClass(SrcReg)) == 16))) {
-    MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg;
-    MCRegister Super = RI.get32BitRegister(RegToFix);
-    assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix);
-    RegToFix = Super;
+  // The rest of copyPhysReg assumes Src and Dst size are the same size.
+  // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
+  // we remove Fix16BitCopies and this code block?
+  if (Fix16BitCopies && ((Size == 16) != (SrcSize == 16))) {
+    // Non-VGPR Src and Dst will later be expanded back to 32 bits.
+    assert(ST.hasTrue16BitInsts());
+    MCRegister &RegToFix = (Size == 32) ? DestReg : SrcReg;
+    MCRegister SubReg = RI.getSubReg(RegToFix, AMDGPU::lo16);
+    RegToFix = SubReg;
 
     if (DestReg == SrcReg) {
-      // Insert empty bundle since ExpandPostRA expects an instruction here.
+      // Identity copy. Insert empty bundle since ExpandPostRA expects an
+      // instruction here.
       BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
       return;
     }
-
     RC = RI.getPhysRegBaseClass(DestReg);
+    Size = RI.getRegSizeInBits(*RC);
+    SrcRC = RI.getPhysRegBaseClass(SrcReg);
+    SrcSize = RI.getRegSizeInBits(*SrcRC);
   }
 
   if (RC == &AMDGPU::VGPR_32RegClass) {
     assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
            AMDGPU::SReg_32RegClass.contains(SrcReg) ||
            AMDGPU::AGPR_32RegClass.contains(SrcReg));
     unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
                      AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
if (RC == &AMDGPU::AGPR_32RegClass) {
     // FIXME: Pass should maintain scavenger to avoid scan through the block on
     // every AGPR spill.
     RegScavenger RS;
     const bool Overlap = RI.regsOverlap(SrcReg, DestReg);
     indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, Overlap);
     return;
   }
 
-  const unsigned Size = RI.getRegSizeInBits(*RC);
   if (Size == 16) {
-    assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
-           AMDGPU::VGPR_HI16RegClass.contains(SrcReg) ||
+    assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
            AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
            AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
 
     bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
     bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
     bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
     bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
     bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) ||
if (IsAGPRDst || IsAGPRSrc) {
         reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
                           "Cannot use hi16 subreg with an AGPR!");
       }
 
       copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);
       return;
     }
 
+    if (ST.hasTrue16BitInsts()) {
+      if (IsSGPRSrc) {
+        assert(SrcLow);
+        SrcReg = NewSrcReg;
+      }
+      // Use the smaller instruction encoding if possible.
+      if (AMDGPU::VGPR_16_Lo128RegClass.contains(DestReg) &&
+          (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(SrcReg))) {
+        BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e32), DestReg)
+            .addReg(SrcReg);
+      } else {
+        BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B16_t16_e64), DestReg)
+            .addImm(0) // src0_modifiers
+            .addReg(SrcReg)
+            .addImm(0); // op_sel
+      }
+      return;
+    }
+
     if (IsSGPRSrc && !ST.hasSDWAScalar()) {
       if (!DstLow || !SrcLow) {
         reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
                           "Cannot use hi16 subreg on VI!");
       }
 
       BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
         .addReg(NewSrcReg, getKillRegState(KillSrc));
auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
       .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
                      : AMDGPU::SDWA::SdwaSel::WORD_1)
       .addReg(NewDestReg, RegState::Implicit | RegState::Undef);
     // First implicit operand is $exec.
     MIB->tieOperands(0, MIB->getNumOperands() - 1);
     return;
   }
 
-  const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(SrcReg);
   if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
     if (ST.hasMovB64()) {
       BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
         .addReg(SrcReg, getKillRegState(KillSrc));
       return;
     }
     if (ST.hasPackedFP32Ops()) {
       BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
 
   return Reg;
 }
 
 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
 
   if (RI.isAGPRClass(DstRC))
     return AMDGPU::COPY;
-  if (RI.getRegSizeInBits(*DstRC) == 32) {
+  if (RI.getRegSizeInBits(*DstRC) == 16) {
+    // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
+    // before RA.
+    return RI.isSGPRClass(DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
+  } else if (RI.getRegSizeInBits(*DstRC) == 32) {
     return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
   } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
     return AMDGPU::S_MOV_B64;
   } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
     return  AMDGPU::V_MOV_B64_PSEUDO;
   }
   return AMDGPU::COPY;
 }

llvm/lib/Target/AMDGPU/VOP1Instructions.td

 }
 
 let SubtargetPredicate = isGFX11Plus in {
   // Restrict src0 to be VGPR
   def V_PERMLANE64_B32 : VOP1_Pseudo<"v_permlane64_b32", VOP_MOVRELS,
                                       getVOP1Pat64<int_amdgcn_permlane64,
                                                    VOP_MOVRELS>.ret,
                                       /*VOP1Only=*/ 1>;
+  defm V_MOV_B16_t16    : VOP1Inst<"v_mov_b16_t16", VOPProfile_True16<VOP_I16_I16>>;
   defm V_NOT_B16        : VOP1Inst_t16<"v_not_b16", VOP_I16_I16>;
   defm V_CVT_I32_I16    : VOP1Inst_t16<"v_cvt_i32_i16", VOP_I32_I16>;
   defm V_CVT_U32_U16    : VOP1Inst_t16<"v_cvt_u32_u16", VOP_I32_I16>;
 } // End SubtargetPredicate = isGFX11Plus
 
 //===----------------------------------------------------------------------===//
 // Target-specific instruction encodings.
 //===----------------------------------------------------------------------===//
defm V_CVT_FLOOR_I32_F32   : VOP1_Real_FULL_with_name_gfx11<0x00d,
   "V_CVT_FLR_I32_F32", "v_cvt_floor_i32_f32">;
 defm V_CLZ_I32_U32         : VOP1_Real_FULL_with_name_gfx11<0x039,
   "V_FFBH_U32", "v_clz_i32_u32">;
 defm V_CTZ_I32_B32         : VOP1_Real_FULL_with_name_gfx11<0x03a,
   "V_FFBL_B32", "v_ctz_i32_b32">;
 defm V_CLS_I32             : VOP1_Real_FULL_with_name_gfx11<0x03b,
   "V_FFBH_I32", "v_cls_i32">;
 defm V_PERMLANE64_B32      : VOP1Only_Real_gfx11<0x067>;
+defm V_MOV_B16_t16           : VOP1_Real_FULL_t16_gfx11<0x01c, "v_mov_b16">;
 defm V_NOT_B16_t16           : VOP1_Real_FULL_t16_gfx11<0x069, "v_not_b16">;
 defm V_CVT_I32_I16_t16       : VOP1_Real_FULL_t16_gfx11<0x06a, "v_cvt_i32_i16">;
 defm V_CVT_U32_U16_t16       : VOP1_Real_FULL_t16_gfx11<0x06b, "v_cvt_u32_u16">;
 
 defm V_CVT_F16_U16_t16       : VOP1_Real_FULL_t16_gfx11<0x050, "v_cvt_f16_u16">;
 defm V_CVT_F16_I16_t16       : VOP1_Real_FULL_t16_gfx11<0x051, "v_cvt_f16_i16">;
 defm V_CVT_U16_F16_t16       : VOP1_Real_FULL_t16_gfx11<0x052, "v_cvt_u16_f16">;
 defm V_CVT_I16_F16_t16       : VOP1_Real_FULL_t16_gfx11<0x053, "v_cvt_i16_f16">;

llvm/test/CodeGen/AMDGPU/lo16-32bit-physreg-copy.mir

-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass postrapseudos -amdgpu-fix-16-bit-physreg-copies -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-
-# GCN-LABEL: name: lo16_to_v32
-# GCN: $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
-name: lo16_to_v32
-tracksRegLiveness: true
-body:             |
-  bb.0:
-    $vgpr0 = IMPLICIT_DEF
-    $vgpr1_lo16 = COPY $vgpr0
-    S_ENDPGM 0
-...
-
-# GCN-LABEL: name: v32_to_lo16
-# GCN: $vgpr1 = V_MOV_B32_e32 $vgpr0, implicit $exec
-name: v32_to_lo16
-tracksRegLiveness: true
-body:             |
-  bb.0:
-    $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = COPY $vgpr0_lo16
-    S_ENDPGM 0
-...
-
-# GCN-LABEL: name: samereg
-# GCN:      $vgpr0 = IMPLICIT_DEF
-# GCN-NEXT: BUNDLE
-# GCN-NEXT: S_ENDPGM
-name: samereg
-tracksRegLiveness: true
-body:             |
-  bb.0:
-    $vgpr0 = IMPLICIT_DEF
-    $vgpr0 = COPY $vgpr0_lo16
-    S_ENDPGM 0
-...