[AMDGPU] Support v_mov_b64 in dpp combine

[foad]

Looks OK.

As an alternative, maybe we could remove V_MOV_B64_DPP_PSEUDO, use the pseudo V_MOV_B64_dpp on all targets, and expand it post-ra only on targets that don't support it?

llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp

169

I wonder why we don't include V_MOV_B32_e64? Maybe we never use it for an immediate copy?

[rampitec]

In D121411#3374893, @foad wrote:

Looks OK.

As an alternative, maybe we could remove V_MOV_B64_DPP_PSEUDO, use the pseudo V_MOV_B64_dpp on all targets, and expand it post-ra only on targets that don't support it?

This maybe a good idea at this point. As a separate change of course.

llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp

169

Probably... Or perhaps it was just forgotten.

[Joe_Nash]

llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp

169

The only reason I can think of to use V_MOV_B32_e64 over V_MOV_B32_e32 is to use operand modifiers, and our backend doesn't even support those for V_MOV_B32_e64 :) But I agree it should be present here, for completeness.

llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp

MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
   if (!Def)
     return nullptr;
 
   switch(Def->getOpcode()) {
   default: break;
   case AMDGPU::IMPLICIT_DEF:
     return nullptr;
   case AMDGPU::COPY:
   case AMDGPU::V_MOV_B32_e32:
-  case AMDGPU::V_MOV_B64_PSEUDO: {
+  case AMDGPU::V_MOV_B64_PSEUDO:
+  case AMDGPU::V_MOV_B64_e32:
+  case AMDGPU::V_MOV_B64_e64: {
     auto &Op1 = Def->getOperand(1);
     if (Op1.isImm())
       return &Op1;
     break;
   }
   }
   return &OldOpnd;
 }
 
 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
                                            MachineInstr &MovMI,
                                            RegSubRegPair CombOldVGPR,
                                            bool CombBCZ,
                                            bool IsShrinkable) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
+         MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
 
   auto OrigOp = OrigMI.getOpcode();
   auto DPPOp = getDPPOp(OrigOp, IsShrinkable);
   if (DPPOp == -1) {
     LLVM_DEBUG(dbgs() << "  failed: no DPP opcode\n");
     return nullptr;
   }
if (!Imm)
     return true;
 
   assert(Imm->isImm());
   return (Imm->getImm() & Mask) == Value;
 }
 
 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
+         MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
          MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
   LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
 
   auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
   assert(DstOpnd && DstOpnd->isReg());
   auto DPPMovReg = DstOpnd->getReg();
   if (DPPMovReg.isPhysical()) {
     LLVM_DEBUG(dbgs() << "  failed: dpp move writes physreg\n");
     return false;
   }
   if (execMayBeModifiedBeforeAnyUse(*MRI, DPPMovReg, MovMI)) {
     LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
                          " for all uses\n");
     return false;
   }
 
-  if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO) {
+  if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
+      MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
     auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
     assert(DppCtrl && DppCtrl->isImm());
     if (!AMDGPU::isLegal64BitDPPControl(DppCtrl->getImm())) {
       LLVM_DEBUG(dbgs() << "  failed: 64 bit dpp move uses unsupported"
                            " control value\n");
       // Let it split, then control may become legal.
       return false;
     }
bool GCNDPPCombine::runOnMachineFunction(MachineFunction &MF) {
   TII = ST->getInstrInfo();
 
   bool Changed = false;
   for (auto &MBB : MF) {
     for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) {
       if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MI)) {
         Changed = true;
         ++NumDPPMovsCombined;
-      } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO) {
+      } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
+                 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
         if (ST->has64BitDPP() && combineDPPMov(MI)) {
           Changed = true;
           ++NumDPPMovsCombined;
         } else {
           auto Split = TII->expandMovDPP64(MI);
           for (auto M : { Split.first, Split.second }) {
             if (M && combineDPPMov(*M))
               ++NumDPPMovsCombined;
           }
           Changed = true;
         }
       }
     }
   }
   return Changed;
 }

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   }
   return true;
 }
 
 std::pair<MachineInstr*, MachineInstr*>
 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
   assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
 
+  if (ST.hasMovB64() &&
+      AMDGPU::isLegal64BitDPPControl(
+        getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
+    MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
+    return std::make_pair(&MI, nullptr);
+  }
+
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MBB.findDebugLoc(MI);
   MachineFunction *MF = MBB.getParent();
   MachineRegisterInfo &MRI = MF->getRegInfo();
   Register Dst = MI.getOperand(0).getReg();
   unsigned Part = 0;
   MachineInstr *Split[2];
 

llvm/test/CodeGen/AMDGPU/dpp64_combine.ll

 ; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP64,GFX90A
+; RUN: llc -march=amdgcn -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64
 ; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10
 
 ; GCN-LABEL: {{^}}dpp64_ceil:
 ; GCN:           global_load_dwordx2 [[V:v\[[0-9:]+\]]],
 ; DPP64:         v_ceil_f64_dpp [[V]], [[V]] row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 ; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_share:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 define amdgpu_kernel void @dpp64_ceil(i64 addrspace(1)* %arg, i64 %in1) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @dpp64_rcp_unsupported_ctl(i64 addrspace(1)* %arg, i64 %in1) {
   %rcp = fdiv fast double 1.0, %tmp1
   %tmp2 = bitcast double %rcp to i64
   store i64 %tmp2, i64 addrspace(1)* %gep
   ret void
 }
 
 ; GCN-LABEL: {{^}}dpp64_div:
 ; GCN:            global_load_dwordx2 [[V:v\[[0-9:]+\]]],
+; DPPMOV64:       v_mov_b64_dpp v[{{[0-9:]+}}], [[V]] row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 ; GFX90A-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 ; GFX10-COUNT-2:  v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_share:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 ; GCN:            v_div_scale_f64
 ; GCN:            v_rcp_f64_e32
 define amdgpu_kernel void @dpp64_div(i64 addrspace(1)* %arg, i64 %in1) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, i64 addrspace(1)* %arg, i32 %id
   %load = load i64, i64 addrspace(1)* %gep

llvm/test/CodeGen/AMDGPU/dpp64_combine.mir

 # RUN: llc -march=amdgcn -mcpu=gfx90a -run-pass=gcn-dpp-combine -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=GCN
+# RUN: llc -march=amdgcn -mcpu=gfx940 -run-pass=gcn-dpp-combine -verify-machineinstrs -o - %s | FileCheck %s --check-prefix=GCN
 
 ---
 # GCN-LABEL: name: dpp64_old_impdef
 # GCN: %3:vreg_64_align2 = V_CEIL_F64_dpp %1, 0, %0, 337, 15, 15, 1, implicit $mode, implicit $exec
 ---
 name: dpp64_old_impdef
 tracksRegLiveness: true
 body: |