diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -1604,6 +1604,18 @@ return true; } +/// Replaces @p CopyMI with an equivalent IMPLICIT_DEF and returns it +static MachineInstr *replaceCopyWithImpDef(MachineInstr *CopyMI, + const TargetInstrInfo *TII) { + CopyMI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); + for (unsigned i = CopyMI->getNumOperands(); i != 0; --i) { + MachineOperand &MO = CopyMI->getOperand(i - 1); + if (MO.isReg() && MO.isUse()) + CopyMI->removeOperand(i - 1); + } + return CopyMI; +} + MachineInstr *RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) { // ProcessImplicitDefs may leave some copies of values, it only // removes local variables. When we have a copy like: @@ -1646,15 +1658,24 @@ // The source interval may also have been on an undef use, in which case the // copy introduced a live value. if (((V && V->isPHIDef()) || (!V && !DstLI.liveAt(Idx)))) { - CopyMI->setDesc(TII->get(TargetOpcode::IMPLICIT_DEF)); - for (unsigned i = CopyMI->getNumOperands(); i != 0; --i) { - MachineOperand &MO = CopyMI->getOperand(i-1); - if (MO.isReg() && MO.isUse()) - CopyMI->removeOperand(i-1); + LLVM_DEBUG(dbgs() << "\tReplacing copy of value with an " + "implicit def\n"); + return replaceCopyWithImpDef(CopyMI, TII); + } + + // Replace undef subreg copies with IMPLICIT_DEF if they're used in + // instructions with early-clobber defs to avoid overlapping register + // assignments in regalloc. + if (DstSubIdx) { + for (const MachineInstr &Use : MRI->use_nodbg_instructions(DstReg)) { + for (const MachineOperand &Def : Use.defs()) { + if (Def.isEarlyClobber()) { + LLVM_DEBUG(dbgs() << "\tReplacing copy of value with an " + "implicit def\n"); + return replaceCopyWithImpDef(CopyMI, TII); + } + } } - LLVM_DEBUG(dbgs() << "\tReplaced copy of value with an " - "implicit def\n"); - return CopyMI; } // Remove any DstReg segments starting at the instruction. diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll @@ -393,6 +393,7 @@ ; GFX11W64-NEXT: s_or_b64 exec, exec, s[6:7] ; GFX11W64-NEXT: s_waitcnt vmcnt(0) ; GFX11W64-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W64-NEXT: ; implicit-def: $sgpr1 ; GFX11W64-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[0:1] @@ -425,6 +426,7 @@ ; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s5 ; GFX11W32-NEXT: s_waitcnt vmcnt(0) ; GFX11W32-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W32-NEXT: ; implicit-def: $sgpr1 ; GFX11W32-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W32-NEXT: v_mad_u64_u32 v[1:2], null, s4, v0, s[0:1] diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll @@ -441,6 +441,7 @@ ; GFX1164-NEXT: .LBB1_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1164-NEXT: v_readfirstlane_b32 s0, v1 +; GFX1164-NEXT: ; implicit-def: $sgpr1 ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164-NEXT: s_mov_b32 s6, -1 @@ -480,6 +481,7 @@ ; GFX1132-NEXT: .LBB1_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v1 +; GFX1132-NEXT: ; implicit-def: $sgpr3 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132-NEXT: s_mov_b32 s6, -1 @@ -1378,14 +1380,16 @@ ; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX1164-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1164-NEXT: v_readfirstlane_b32 s3, v1 +; GFX1164-NEXT: ; implicit-def: $vgpr4 ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164-NEXT: s_mov_b32 s6, -1 ; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, s0, v2, s[2:3] -; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s1, v2, v[1:2] -; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164-NEXT: v_mov_b32_e32 v1, v3 +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s1, v2, v[3:4] +; GFX1164-NEXT: v_mov_b32_e32 v1, v5 ; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX1164-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1164-NEXT: s_endpgm @@ -1425,14 +1429,16 @@ ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1132-NEXT: v_readfirstlane_b32 s3, v1 +; GFX1132-NEXT: ; implicit-def: $vgpr4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132-NEXT: s_mov_b32 s6, -1 ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, s0, v2, s[2:3] -; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s1, v2, v[1:2] -; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-NEXT: v_mov_b32_e32 v3, v1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s1, v2, v[3:4] +; GFX1132-NEXT: v_mov_b32_e32 v1, v5 ; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX1132-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1132-NEXT: s_endpgm @@ -3007,15 +3013,16 @@ ; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s0, v2, 0 +; GFX1164-NEXT: ; implicit-def: $vgpr5 ; GFX1164-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1164-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164-NEXT: s_mov_b32 s6, -1 ; GFX1164-NEXT: s_waitcnt_depctr 0xfff -; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s1, v2, v[4:5] +; GFX1164-NEXT: v_mad_u64_u32 v[6:7], null, s1, v2, v[4:5] ; GFX1164-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s0, v3 ; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-NEXT: v_mov_b32_e32 v1, v5 +; GFX1164-NEXT: v_mov_b32_e32 v1, v6 ; GFX1164-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s1, v1, vcc ; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX1164-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3056,15 +3063,16 @@ ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s2 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s0, v2, 0 +; GFX1132-NEXT: ; implicit-def: $vgpr5 ; GFX1132-NEXT: v_readfirstlane_b32 s0, v0 ; GFX1132-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132-NEXT: s_mov_b32 s6, -1 ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4) -; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s1, v2, v[4:5] +; GFX1132-NEXT: v_mad_u64_u32 v[6:7], null, s1, v2, v[4:5] ; GFX1132-NEXT: v_readfirstlane_b32 s1, v1 ; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s0, v3 ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-NEXT: v_mov_b32_e32 v1, v5 +; GFX1132-NEXT: v_mov_b32_e32 v1, v6 ; GFX1132-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo ; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0 ; GFX1132-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll @@ -430,6 +430,7 @@ ; GFX1164-NEXT: .LBB1_2: ; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1] ; GFX1164-NEXT: v_readfirstlane_b32 s0, v1 +; GFX1164-NEXT: ; implicit-def: $sgpr1 ; GFX1164-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) @@ -465,6 +466,7 @@ ; GFX1132-NEXT: .LBB1_2: ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v1 +; GFX1132-NEXT: ; implicit-def: $sgpr3 ; GFX1132-NEXT: s_mov_b32 s7, 0x31016000 ; GFX1132-NEXT: s_mov_b32 s6, -1 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) @@ -1537,14 +1539,16 @@ ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: v_readfirstlane_b32 s4, v0 ; GFX1164-NEXT: v_readfirstlane_b32 s5, v1 +; GFX1164-NEXT: ; implicit-def: $vgpr4 ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5] ; GFX1164-NEXT: s_mov_b32 s2, -1 -; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s3, v2, v[3:4] ; GFX1164-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1164-NEXT: v_mov_b32_e32 v1, v3 +; GFX1164-NEXT: v_mov_b32_e32 v1, v5 ; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0 ; GFX1164-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1164-NEXT: s_endpgm @@ -1578,14 +1582,16 @@ ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: v_readfirstlane_b32 s4, v0 ; GFX1132-NEXT: v_readfirstlane_b32 s5, v1 +; GFX1132-NEXT: ; implicit-def: $vgpr4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5] ; GFX1132-NEXT: s_mov_b32 s2, -1 -; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2] +; GFX1132-NEXT: v_mov_b32_e32 v3, v1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s3, v2, v[3:4] ; GFX1132-NEXT: s_mov_b32 s3, 0x31016000 -; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1132-NEXT: v_mov_b32_e32 v1, v3 +; GFX1132-NEXT: v_mov_b32_e32 v1, v5 ; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0 ; GFX1132-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1132-NEXT: s_endpgm @@ -3229,15 +3235,16 @@ ; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX1164-NEXT: s_waitcnt lgkmcnt(0) ; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s2, v2, 0 +; GFX1164-NEXT: ; implicit-def: $vgpr5 ; GFX1164-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1164-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1164-NEXT: s_waitcnt_depctr 0xfff -; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s3, v2, v[4:5] +; GFX1164-NEXT: v_mad_u64_u32 v[6:7], null, s3, v2, v[4:5] ; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v3 ; GFX1164-NEXT: s_mov_b32 s3, 0x31016000 ; GFX1164-NEXT: s_mov_b32 s2, -1 ; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1164-NEXT: v_mov_b32_e32 v1, v5 +; GFX1164-NEXT: v_mov_b32_e32 v1, v6 ; GFX1164-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s4, v1, vcc ; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0 ; GFX1164-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3272,15 +3279,16 @@ ; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4 ; GFX1132-NEXT: s_waitcnt lgkmcnt(0) ; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s2, v2, 0 +; GFX1132-NEXT: ; implicit-def: $vgpr5 ; GFX1132-NEXT: v_readfirstlane_b32 s2, v0 ; GFX1132-NEXT: v_readfirstlane_b32 s4, v1 ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s3, v2, v[4:5] +; GFX1132-NEXT: v_mad_u64_u32 v[6:7], null, s3, v2, v[4:5] ; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v3 ; GFX1132-NEXT: s_mov_b32 s3, 0x31016000 ; GFX1132-NEXT: s_mov_b32 s2, -1 ; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1132-NEXT: v_mov_b32_e32 v1, v5 +; GFX1132-NEXT: v_mov_b32_e32 v1, v6 ; GFX1132-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s4, v1, vcc_lo ; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0 ; GFX1132-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll @@ -392,6 +392,7 @@ ; GFX11W64-NEXT: s_or_b64 exec, exec, s[6:7] ; GFX11W64-NEXT: s_waitcnt vmcnt(0) ; GFX11W64-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W64-NEXT: ; implicit-def: $sgpr1 ; GFX11W64-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[0:1] @@ -424,6 +425,7 @@ ; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s5 ; GFX11W32-NEXT: s_waitcnt vmcnt(0) ; GFX11W32-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W32-NEXT: ; implicit-def: $sgpr1 ; GFX11W32-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W32-NEXT: v_mad_u64_u32 v[1:2], null, s4, v0, s[0:1] diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll --- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll +++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll @@ -405,6 +405,7 @@ ; GFX11W64-NEXT: s_or_b64 exec, exec, s[4:5] ; GFX11W64-NEXT: s_waitcnt vmcnt(0) ; GFX11W64-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W64-NEXT: ; implicit-def: $sgpr1 ; GFX11W64-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[0:1] @@ -438,6 +439,7 @@ ; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s5 ; GFX11W32-NEXT: s_waitcnt vmcnt(0) ; GFX11W32-NEXT: v_readfirstlane_b32 s0, v1 +; GFX11W32-NEXT: ; implicit-def: $sgpr1 ; GFX11W32-NEXT: s_waitcnt lgkmcnt(0) ; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11W32-NEXT: v_mad_u64_u32 v[1:2], null, s4, v0, s[0:1] diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-early-clobber-undef-subreg.mir b/llvm/test/CodeGen/AMDGPU/coalescer-early-clobber-undef-subreg.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/coalescer-early-clobber-undef-subreg.mir @@ -0,0 +1,26 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-coalescing -run-pass=simple-register-coalescing -o - %s | FileCheck %s + +# Check for subreg IMPLICIT_DEF if an undef subreg copy is used in instructions +# with early-clobber defs +--- +name: test +tracksRegLiveness: true +registers: + - { id: 1, class: vgpr_32 } +body: | + bb.0: + liveins: $vgpr0, $sgpr0_sgpr1 + ; CHECK-LABEL: name: test + ; CHECK: liveins: $vgpr0, $sgpr0_sgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: undef %2.sub0:vreg_64 = COPY $vgpr0 + ; CHECK-NEXT: %2.sub1:vreg_64 = IMPLICIT_DEF + ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec = COPY $sgpr0_sgpr1 + ; CHECK-NEXT: dead early-clobber %4:vreg_64, dead %5:sreg_32 = V_MAD_U64_U32_gfx11_e64 [[COPY]].sub1, [[COPY]].sub0, %2, 0, implicit $exec + %0:vgpr_32 = COPY $vgpr0 + undef %2.sub0:vreg_64 = COPY %0 + %2.sub1:vreg_64 = COPY undef %1 + %3:sreg_64_xexec = COPY $sgpr0_sgpr1 + early-clobber %4:vreg_64, %5:sreg_32 = V_MAD_U64_U32_gfx11_e64 %3.sub1:sreg_64_xexec, %3.sub0:sreg_64_xexec, %2:vreg_64, 0, implicit $exec +... diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll --- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll @@ -400,12 +400,13 @@ ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0 +; GFX11-NEXT: ; implicit-def: $vgpr6 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3] -; GFX11-NEXT: v_ashrrev_i32_e32 v5, 31, v5 -; GFX11-NEXT: v_mov_b32_e32 v3, v1 +; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v5 +; GFX11-NEXT: v_mov_b32_e32 v5, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v5, v4, v[3:4] +; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v3, v4, v[5:6] ; GFX11-NEXT: s_setpc_b64 s[30:31] %ext0 = sext i32 %arg0 to i64 %ext1 = zext i32 %arg1 to i64 @@ -489,11 +490,13 @@ ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v0 ; GFX11-NEXT: v_mov_b32_e32 v6, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v2, v3, v[4:5] -; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_and_b32 v5, 1, v6 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[4:5] +; GFX11-NEXT: v_and_b32_e32 v6, 1, v6 +; GFX11-NEXT: ; implicit-def: $vgpr5 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v3, v[4:5] ; GFX11-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 8589934591 %trunc.rhs = and i64 %arg1, 4294967295 @@ -541,11 +544,13 @@ ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX11-NEXT: v_mov_b32_e32 v6, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v6, v2, v[4:5] -; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v4, 1, v3 +; GFX11-NEXT: v_and_b32_e32 v5, 1, v3 +; GFX11-NEXT: ; implicit-def: $vgpr4 +; GFX11-NEXT: v_mov_b32_e32 v3, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v4, v[3:4] +; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, v6, v5, v[3:4] ; GFX11-NEXT: s_setpc_b64 s[30:31] %trunc.lhs = and i64 %arg0, 4294967295 %trunc.rhs = and i64 %arg1, 8589934591 diff --git a/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll b/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll --- a/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_u64_u32.ll @@ -18,9 +18,10 @@ ; ; GFX11-LABEL: mad_i32_vvv: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mov_b32_e32 v3, v1 -; GFX11-NEXT: v_mov_b32_e32 v4, v0 -; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v4, v3, v[2:3] +; GFX11-NEXT: v_mov_b32_e32 v4, v1 +; GFX11-NEXT: v_mov_b32_e32 v5, v0 +; GFX11-NEXT: ; implicit-def: $vgpr3 +; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v5, v4, v[2:3] ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b %add = add i32 %mul, %c @@ -127,8 +128,9 @@ ; ; GFX11-LABEL: mad_i32_vcv: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v0, 42, v[1:2] -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: ; implicit-def: $vgpr2 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v0, 42, v[1:2] +; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, 42 %add = add i32 %mul, %c @@ -173,6 +175,7 @@ ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, v1 ; GFX11-NEXT: v_mov_b32_e32 v3, v0 +; GFX11-NEXT: ; implicit-def: $sgpr1 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, s[0:1] ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b @@ -194,8 +197,9 @@ ; ; GFX11-LABEL: mad_i32_vsv: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, v0, s0, v[1:2] -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: ; implicit-def: $vgpr2 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, v0, s0, v[1:2] +; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b %add = add i32 %mul, %c @@ -216,8 +220,9 @@ ; ; GFX11-LABEL: mad_i32_svv: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, s0, v0, v[1:2] -; GFX11-NEXT: v_mov_b32_e32 v0, v2 +; GFX11-NEXT: ; implicit-def: $vgpr2 +; GFX11-NEXT: v_mad_u64_u32 v[3:4], null, s0, v0, v[1:2] +; GFX11-NEXT: v_mov_b32_e32 v0, v3 ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b %add = add i32 %mul, %c @@ -242,6 +247,7 @@ ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, v0 ; GFX11-NEXT: s_mov_b32 s2, s1 +; GFX11-NEXT: ; implicit-def: $sgpr3 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, v2, s0, s[2:3] ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b @@ -267,6 +273,7 @@ ; GFX11: ; %bb.0: ; GFX11-NEXT: v_mov_b32_e32 v2, v0 ; GFX11-NEXT: s_mov_b32 s2, s1 +; GFX11-NEXT: ; implicit-def: $sgpr3 ; GFX11-NEXT: v_mad_u64_u32 v[0:1], null, s0, v2, s[2:3] ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b @@ -289,8 +296,9 @@ ; ; GFX11-LABEL: mad_i32_ssv: ; GFX11: ; %bb.0: -; GFX11-NEXT: v_mad_u64_u32 v[1:2], null, s0, s1, v[0:1] -; GFX11-NEXT: v_mov_b32_e32 v0, v1 +; GFX11-NEXT: ; implicit-def: $vgpr1 +; GFX11-NEXT: v_mad_u64_u32 v[2:3], null, s0, s1, v[0:1] +; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: ; return to shader part epilog %mul = mul i32 %a, %b %add = add i32 %mul, %c diff --git a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll --- a/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll +++ b/llvm/test/CodeGen/Thumb2/mve-laneinterleaving-cost.ll @@ -374,16 +374,20 @@ ; CHECK-NEXT: vldrw.u32 q1, [r1] ; CHECK-NEXT: ldr.w lr, [sp, #20] ; CHECK-NEXT: vmov.f32 s10, s5 +; CHECK-NEXT: @ implicit-def: $s5 ; CHECK-NEXT: vmov.f32 s14, s1 +; CHECK-NEXT: @ implicit-def: $s1 ; CHECK-NEXT: vmov r5, s0 ; CHECK-NEXT: vmov.f32 s0, s2 ; CHECK-NEXT: vmov.f32 s2, s3 +; CHECK-NEXT: @ implicit-def: $s3 ; CHECK-NEXT: vmov r0, s10 ; CHECK-NEXT: vmov r1, s14 ; CHECK-NEXT: smull r12, r3, r1, r0 ; CHECK-NEXT: vmov r0, s4 ; CHECK-NEXT: vmov.f32 s4, s6 ; CHECK-NEXT: vmov.f32 s6, s7 +; CHECK-NEXT: @ implicit-def: $s7 ; CHECK-NEXT: vmullb.s32 q2, q0, q1 ; CHECK-NEXT: asrl r12, r3, r2 ; CHECK-NEXT: vmov r6, r1, d4 diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll --- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll +++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll @@ -223,10 +223,14 @@ ; CHECK-NEXT: add.w r9, r5, r1, lsl #2 ; CHECK-NEXT: add.w r12, r0, r1, lsl #2 ; CHECK-NEXT: vldrw.u32 q1, [r4] +; CHECK-NEXT: @ implicit-def: $s21 +; CHECK-NEXT: @ implicit-def: $s23 ; CHECK-NEXT: .LBB1_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q3, [r5], #16 ; CHECK-NEXT: vldrw.u32 q2, [r0], #16 +; CHECK-NEXT: @ implicit-def: $s17 +; CHECK-NEXT: @ implicit-def: $s19 ; CHECK-NEXT: str r2, [sp, #12] @ 4-byte Spill ; CHECK-NEXT: mov.w r2, #-1 ; CHECK-NEXT: vmov.f32 s16, s10 @@ -457,11 +461,15 @@ ; CHECK-NEXT: vldrw.u32 q3, [r5] ; CHECK-NEXT: vdup.32 q1, r6 ; CHECK-NEXT: mvn r8, #-2147483648 +; CHECK-NEXT: @ implicit-def: $s29 +; CHECK-NEXT: @ implicit-def: $s31 ; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill ; CHECK-NEXT: .LBB2_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload ; CHECK-NEXT: vdup.32 q4, r9 +; CHECK-NEXT: @ implicit-def: $s25 +; CHECK-NEXT: @ implicit-def: $s27 ; CHECK-NEXT: movs r4, #0 ; CHECK-NEXT: add.w r9, r9, #4 ; CHECK-NEXT: vorr q4, q4, q0 @@ -790,10 +798,16 @@ ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q1, [r0], #16 ; CHECK-NEXT: vldrw.u32 q2, [r1], #16 +; CHECK-NEXT: @ implicit-def: $s13 +; CHECK-NEXT: @ implicit-def: $s15 +; CHECK-NEXT: @ implicit-def: $s17 +; CHECK-NEXT: @ implicit-def: $s19 ; CHECK-NEXT: vmov.f32 s12, s6 -; CHECK-NEXT: vmov.f32 s16, s10 ; CHECK-NEXT: vmov.f32 s14, s7 +; CHECK-NEXT: @ implicit-def: $s7 +; CHECK-NEXT: vmov.f32 s16, s10 ; CHECK-NEXT: vmov.f32 s18, s11 +; CHECK-NEXT: @ implicit-def: $s11 ; CHECK-NEXT: vmullb.u32 q5, q4, q3 ; CHECK-NEXT: vmov.f32 s6, s5 ; CHECK-NEXT: vmov r10, r5, d10 diff --git a/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll b/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll --- a/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vmull-splat.ll @@ -194,6 +194,8 @@ ; CHECK-NEXT: vmullb.s32 q2, q0, q3 ; CHECK-NEXT: vmov.f32 s0, s1 ; CHECK-NEXT: vmov.f32 s2, s3 +; CHECK-NEXT: @ implicit-def: $s1 +; CHECK-NEXT: @ implicit-def: $s3 ; CHECK-NEXT: vmullb.s32 q1, q0, q3 ; CHECK-NEXT: vmov q0, q2 ; CHECK-NEXT: bx lr @@ -214,6 +216,8 @@ ; CHECK-NEXT: vmullb.s32 q2, q3, q0 ; CHECK-NEXT: vmov.f32 s0, s1 ; CHECK-NEXT: vmov.f32 s2, s3 +; CHECK-NEXT: @ implicit-def: $s1 +; CHECK-NEXT: @ implicit-def: $s3 ; CHECK-NEXT: vmullb.s32 q1, q3, q0 ; CHECK-NEXT: vmov q0, q2 ; CHECK-NEXT: bx lr @@ -470,6 +474,8 @@ ; CHECK-NEXT: vmullb.u32 q2, q0, q3 ; CHECK-NEXT: vmov.f32 s0, s1 ; CHECK-NEXT: vmov.f32 s2, s3 +; CHECK-NEXT: @ implicit-def: $s1 +; CHECK-NEXT: @ implicit-def: $s3 ; CHECK-NEXT: vmullb.u32 q1, q0, q3 ; CHECK-NEXT: vmov q0, q2 ; CHECK-NEXT: bx lr @@ -490,6 +496,8 @@ ; CHECK-NEXT: vmullb.u32 q2, q3, q0 ; CHECK-NEXT: vmov.f32 s0, s1 ; CHECK-NEXT: vmov.f32 s2, s3 +; CHECK-NEXT: @ implicit-def: $s1 +; CHECK-NEXT: @ implicit-def: $s3 ; CHECK-NEXT: vmullb.u32 q1, q3, q0 ; CHECK-NEXT: vmov q0, q2 ; CHECK-NEXT: bx lr