[AMDGPU] Fix shrinking of F16 FMA on newer subtargets

[foad]

Summary

D125803 introduced shrinking of F16 FMA to FMAAK/FMAMK in
SIShrinkInstructions (useful on GFX10+ where VOP3 instructions may have
a literal operand) but failed to handle the V_FMA_F16_gfx9_e64 form of
the opcode which is used on GFX9+.

[Joe_Nash]

LGTM

llvm/test/CodeGen/AMDGPU/gfx10-shrink-mad-fma.mir

195

I guess the test case before was technically invalid, because V_FMA_F16_e64 should not exist on GFX10, only GFX8. But afaik we do not check whether an instruction is valid on the subtarget in SIInstrInfo::verifyInstruction.

[foad]

llvm/test/CodeGen/AMDGPU/gfx10-shrink-mad-fma.mir

195

Right. See https://reviews.llvm.org/D129506#inline-1246267.

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
   default:
     return false;
   }
 }
 
 static constexpr unsigned ModifierOpNames[] = {
     AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
     AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
-    AMDGPU::OpName::omod};
+    AMDGPU::OpName::omod,           AMDGPU::OpName::op_sel};
 
 void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
   unsigned Opc = MI.getOpcode();
-  for (unsigned Name : reverse(ModifierOpNames))
-    MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, Name));
+  for (unsigned Name : reverse(ModifierOpNames)) {
+    int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);
+    if (Idx >= 0)
+      MI.removeOperand(Idx);
+  }
 }
 
 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
                                 Register Reg, MachineRegisterInfo *MRI) const {
   if (!MRI->hasOneNonDBGUse(Reg))
     return false;
 
   switch (DefMI.getOpcode()) {

llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp

case AMDGPU::V_MAD_F32_e64:
       break;
     case AMDGPU::V_FMA_F32_e64:
       NewOpcode = AMDGPU::V_FMAAK_F32;
       break;
     case AMDGPU::V_MAD_F16_e64:
       NewOpcode = AMDGPU::V_MADAK_F16;
       break;
     case AMDGPU::V_FMA_F16_e64:
+    case AMDGPU::V_FMA_F16_gfx9_e64:
       NewOpcode = AMDGPU::V_FMAAK_F16;
       break;
     }
   }
 
   // Detect "Dst = VSrc * Imm + VGPR" and convert to MK form.
   if (Src2.isReg() && TRI->isVGPR(*MRI, Src2.getReg())) {
     if (Src1.isImm() && !TII->isInlineConstant(Src1))
case AMDGPU::V_MAD_F32_e64:
       break;
     case AMDGPU::V_FMA_F32_e64:
       NewOpcode = AMDGPU::V_FMAMK_F32;
       break;
     case AMDGPU::V_MAD_F16_e64:
       NewOpcode = AMDGPU::V_MADMK_F16;
       break;
     case AMDGPU::V_FMA_F16_e64:
+    case AMDGPU::V_FMA_F16_gfx9_e64:
       NewOpcode = AMDGPU::V_FMAMK_F16;
       break;
     }
   }
 
   if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END)
     return;
 
for (I = MBB.begin(); I != MBB.end(); I = Next) {
       }
 
       if (!TII->isVOP3(MI))
         continue;
 
       if (MI.getOpcode() == AMDGPU::V_MAD_F32_e64 ||
           MI.getOpcode() == AMDGPU::V_FMA_F32_e64 ||
           MI.getOpcode() == AMDGPU::V_MAD_F16_e64 ||
-          MI.getOpcode() == AMDGPU::V_FMA_F16_e64) {
+          MI.getOpcode() == AMDGPU::V_FMA_F16_e64 ||
+          MI.getOpcode() == AMDGPU::V_FMA_F16_gfx9_e64) {
         shrinkMadFma(MI);
         continue;
       }
 
       if (!TII->hasVALU32BitEncoding(MI.getOpcode())) {
         // If there is no chance we will shrink it and use VCC as sdst to get
         // a 32 bit form try to replace dead sdst with NULL.
         tryReplaceDeadSDST(MI);

llvm/test/CodeGen/AMDGPU/fma.f16.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-SDAG
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL
-; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-SDAG
-; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10
 
 declare half @llvm.fma.f16(half, half, half)
 declare half @llvm.maxnum.f16(half, half)
 
 define half @test_fma(half %x, half %y, half %z) {
 ; GFX9-LABEL: test_fma:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)

 ;
 ; GFX9-GISEL-LABEL: test_fmaak:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0x4200
 ; GFX9-GISEL-NEXT:    v_fma_f16 v0, v0, v1, v2
 ; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-SDAG-LABEL: test_fmaak:
-; GFX10-SDAG:       ; %bb.0:
-; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-SDAG-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
-; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-GISEL-LABEL: test_fmaak:
-; GFX10-GISEL:       ; %bb.0:
-; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-GISEL-NEXT:    v_fma_f16 v0, v0, v1, 0x4200
-; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: test_fmaak:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half %y, half 0xH4200)
   ret half %r
 }
 
 ; GFX10+ has v_fmamk_f16.
 define half @test_fmamk(half %x, half %y, half %z) {
 ; GFX9-SDAG-LABEL: test_fmamk:
 ; GFX9-SDAG:       ; %bb.0:

llvm/test/CodeGen/AMDGPU/gfx10-shrink-mad-fma.mir

body: |
   bb.0:
     ; GFX10-LABEL: name: fma_cvv_f16
     ; GFX10: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[V_FMAMK_F16_:%[0-9]+]]:vgpr_32 = V_FMAMK_F16 [[DEF]], 18688, [[DEF1]], implicit $mode, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit [[V_FMAMK_F16_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
-    %2:vgpr_32 = V_FMA_F16_e64 0, 18688, 0, %0, 0, %1, 0, 0, implicit $mode, implicit $exec
+    %2:vgpr_32 = V_FMA_F16_gfx9_e64 0, 18688, 0, %0, 0, %1, 0, 0, implicit $mode, implicit $exec
     SI_RETURN implicit %2
 ...
 
 ---
 name: fma_vcv_f16
 body: |
   bb.0:
     ; GFX10-LABEL: name: fma_vcv_f16
     ; GFX10: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[V_FMAMK_F16_:%[0-9]+]]:vgpr_32 = V_FMAMK_F16 [[DEF]], 18688, [[DEF1]], implicit $mode, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit [[V_FMAMK_F16_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
-    %2:vgpr_32 = V_FMA_F16_e64 0, %0, 0, 18688, 0, %1, 0, 0, implicit $mode, implicit $exec
+    %2:vgpr_32 = V_FMA_F16_gfx9_e64 0, %0, 0, 18688, 0, %1, 0, 0, implicit $mode, implicit $exec
     SI_RETURN implicit %2
 ...
 
 ---
 name: fma_vvc_f16
 body: |
   bb.0:
     ; GFX10-LABEL: name: fma_vvc_f16
     ; GFX10: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[V_FMAAK_F16_:%[0-9]+]]:vgpr_32 = V_FMAAK_F16 [[DEF]], [[DEF1]], 18688, implicit $mode, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit [[V_FMAAK_F16_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
-    %2:vgpr_32 = V_FMA_F16_e64 0, %0, 0, %1, 0, 18688, 0, 0, implicit $mode, implicit $exec
+    %2:vgpr_32 = V_FMA_F16_gfx9_e64 0, %0, 0, %1, 0, 18688, 0, 0, implicit $mode, implicit $exec
     SI_RETURN implicit %2
 ...
 
 ---
 name: fma_vsc_f16
 body: |
   bb.0:
     ; GFX10-LABEL: name: fma_vsc_f16
     ; GFX10: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
     ; GFX10-NEXT: [[V_FMAAK_F16_:%[0-9]+]]:vgpr_32 = V_FMAAK_F16 [[DEF1]], [[DEF]], 18688, implicit $mode, implicit $exec
     ; GFX10-NEXT: SI_RETURN implicit [[V_FMAAK_F16_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:sreg_32 = IMPLICIT_DEF
-    %2:vgpr_32 = V_FMA_F16_e64 0, %0, 0, %1, 0, 18688, 0, 0, implicit $mode, implicit $exec
+    %2:vgpr_32 = V_FMA_F16_gfx9_e64 0, %0, 0, %1, 0, 18688, 0, 0, implicit $mode, implicit $exec
     SI_RETURN implicit %2
 ...