diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -363,6 +363,10 @@ } unsigned NumWaitStates = TII.getNumWaitStates(*CurrCycleInstr); + if (!NumWaitStates) { + CurrCycleInstr = nullptr; + return; + } // Keep track of emitted instructions EmittedInstrs.push_front(CurrCycleInstr); diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -1641,6 +1641,13 @@ case AMDGPU::S_NOP: return MI.getOperand(0).getImm() + 1; + + // FIXME: Any other pseudo instruction? + // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The + // hazard, even if one exist, won't really be visible. Should we handle it? + case AMDGPU::SI_MASKED_UNREACHABLE: + case AMDGPU::WAVE_BARRIER: + return 0; } } diff --git a/llvm/test/CodeGen/AMDGPU/hazard-pseudo-machineinstrs.mir b/llvm/test/CodeGen/AMDGPU/hazard-pseudo-machineinstrs.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/hazard-pseudo-machineinstrs.mir @@ -0,0 +1,45 @@ +# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs -run-pass post-RA-sched %s -o - | FileCheck -check-prefix=GCN %s + +# WAVE_BARRIER and SI_MASKED_UNREACHABLE are not really instructions. +# To fix the hazard (m0 def followed by V_INTERP), the scheduler +# should move another instruction into the slot. +--- +# CHECK-LABEL: name: hazard_wave_barrier +# CHECK-LABEL: bb.0: +# GCN: $m0 = S_MOV_B32 killed renamable $sgpr0 +# GCN-NEXT: WAVE_BARRIER +# GCN-NEXT: S_MOV_B32 0 +# GCN-NEXT: V_INTERP_MOV_F32 +name: hazard_wave_barrier +tracksRegLiveness: true +body: | + bb.0: + liveins: $sgpr0 + + $m0 = S_MOV_B32 killed renamable $sgpr0 + WAVE_BARRIER + renamable $vgpr0 = V_INTERP_MOV_F32 2, 0, 0, implicit $mode, implicit $m0, implicit $exec + renamable $sgpr1 = S_MOV_B32 0 + S_ENDPGM 0 + +... +# GCN-LABEL: name: hazard-masked-unreachable +# CHECK-LABEL: bb.0: +# GCN: $m0 = S_MOV_B32 killed renamable $sgpr0 +# GCN-NEXT: SI_MASKED_UNREACHABLE +# GCN-NEXT: S_MOV_B32 0 +# GCN-NEXT: V_INTERP_MOV_F32 +--- +name: hazard-masked-unreachable +tracksRegLiveness: true +body: | + bb.0: + liveins: $sgpr0 + + $m0 = S_MOV_B32 killed renamable $sgpr0 + SI_MASKED_UNREACHABLE + renamable $vgpr0 = V_INTERP_MOV_F32 2, 0, 0, implicit $mode, implicit $m0, implicit $exec + renamable $sgpr1 = S_MOV_B32 0 + bb.1: + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/hazard.mir b/llvm/test/CodeGen/AMDGPU/hazard.mir --- a/llvm/test/CodeGen/AMDGPU/hazard.mir +++ b/llvm/test/CodeGen/AMDGPU/hazard.mir @@ -125,3 +125,49 @@ S_SENDMSG 3, implicit $exec, implicit $m0 S_ENDPGM 0 ... +# GCN-LABEL: name: hazard-lookahead-wave-barrier +# GCN: S_WAITCNT 0 +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: V_ADD_F16_dpp +--- +name: hazard-lookahead-wave-barrier +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr3 + + renamable $vgpr1 = contract nofpexcept V_ADD_F16_e32 killed $vgpr1, $vgpr0, implicit $mode, implicit $exec + WAVE_BARRIER + S_WAITCNT 0 + renamable $vgpr2 = contract nofpexcept V_ADD_F16_dpp undef $vgpr2, 0, $vgpr1, 0, $vgpr3, 273, 15, 15, 1, implicit $mode, implicit $exec +... +# GCN-LABEL: name: hazard-lookahead-masked-unreachable +# GCN: SI_MASKED_UNREACHABLE +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: S_SENDMSG +--- +name: hazard-lookahead-masked-unreachable +body: | + bb.0: + $m0 = S_MOV_B32 -1 + SI_MASKED_UNREACHABLE + S_SENDMSG 3, implicit $exec, implicit $m0 + + bb.1: + S_ENDPGM 0 +... +# GCN-LABEL: name: fallthrough-hazard-lookahead-masked-unreachable +# GCN: SI_MASKED_UNREACHABLE +# GCN-LABEL: bb.1: +# GCN-NEXT: S_NOP 0 +# GCN-NEXT: S_SENDMSG +--- +name: fallthrough-hazard-lookahead-masked-unreachable +body: | + bb.0: + $m0 = S_MOV_B32 -1 + SI_MASKED_UNREACHABLE + + bb.1: + S_SENDMSG 3, implicit $exec, implicit $m0 + S_ENDPGM 0 +...