diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp --- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp @@ -57,10 +57,14 @@ unsigned SkipThreshold = 0; MachineDominatorTree *MDT = nullptr; + MachineBasicBlock *EmptyMBBAtEnd = nullptr; + MachineBasicBlock *EarlyExitBlock = nullptr; + bool shouldSkip(const MachineBasicBlock &From, const MachineBasicBlock &To) const; bool dominatesAllReachable(MachineBasicBlock &MBB); + void createEarlyExitBlock(MachineBasicBlock &MBB); void skipIfDead(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL); @@ -161,17 +165,46 @@ return true; } +static void generateEndPgm(MachineBasicBlock *MBB, DebugLoc DL, + const SIInstrInfo *TII) { + // Generate "null export; s_endpgm". + BuildMI(MBB, DL, TII->get(AMDGPU::EXP_DONE)) + .addImm(0x09) // V_008DFC_SQ_EXP_NULL + .addReg(AMDGPU::VGPR0, RegState::Undef) + .addReg(AMDGPU::VGPR0, RegState::Undef) + .addReg(AMDGPU::VGPR0, RegState::Undef) + .addReg(AMDGPU::VGPR0, RegState::Undef) + .addImm(1) // vm + .addImm(0) // compr + .addImm(0); // en + BuildMI(MBB, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0); +} + +void SIInsertSkips::createEarlyExitBlock(MachineBasicBlock &MBB) { + MachineFunction *MF = MBB.getParent(); + DebugLoc DL; + + assert(!EarlyExitBlock); + EarlyExitBlock = MF->CreateMachineBasicBlock(); + + // If an empty block exists at the end of the shader to allow for appending + // an epilog then the early exit block should come before it. + if (EmptyMBBAtEnd) { + MF->insert(EmptyMBBAtEnd->getIterator(), EarlyExitBlock); + } else { + MF->insert(MF->end(), EarlyExitBlock); + } + + generateEndPgm(EarlyExitBlock, DL, TII); +} + /// Insert an "if exec=0 { null export; s_endpgm }" sequence before the given /// iterator. Only applies to pixel shaders. void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL) { MachineFunction *MF = MBB.getParent(); assert(MF->getFunction().getCallingConv() == CallingConv::AMDGPU_PS); - - // Currently, SI_KILL_*_TERMINATOR is expected to occur only as the last - // terminator of a basic block. If this ever changes, we need to optionally - // split MBB here. - assert(I == MBB.end()); + (void)MF; // It is possible for an SI_KILL_*_TERMINATOR to sit at the bottom of a // basic block that has no further successors (e.g., there was an @@ -187,33 +220,17 @@ // already-existing basic block. auto NextBBI = std::next(MBB.getIterator()); bool NoSuccessor = llvm::find(MBB.successors(), &*NextBBI) == MBB.succ_end(); - MachineBasicBlock *SkipBB; if (NoSuccessor) { - SkipBB = &MBB; + generateEndPgm(&MBB, DL, TII); } else { - // Create a new basic block that will contain the "null export; s_endpgm" - // and set up the branching to go around it. - SkipBB = MF->CreateMachineBasicBlock(); - MF->insert(NextBBI, SkipBB); - - BuildMI(&MBB, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&*NextBBI); - MBB.addSuccessor(SkipBB); + if (!EarlyExitBlock) + createEarlyExitBlock(MBB); - MDT->addNewBlock(SkipBB, &MBB); + BuildMI(&MBB, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)).addMBB(EarlyExitBlock); + MBB.addSuccessor(EarlyExitBlock); + MDT->getBase().insertEdge(&MBB, EarlyExitBlock); } - - // Generate "null export; s_endpgm". - BuildMI(SkipBB, DL, TII->get(AMDGPU::EXP_DONE)) - .addImm(0x09) // V_008DFC_SQ_EXP_NULL - .addReg(AMDGPU::VGPR0, RegState::Undef) - .addReg(AMDGPU::VGPR0, RegState::Undef) - .addReg(AMDGPU::VGPR0, RegState::Undef) - .addReg(AMDGPU::VGPR0, RegState::Undef) - .addImm(1) // vm - .addImm(0) // compr - .addImm(0); // en - BuildMI(SkipBB, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0); } /// Translate a SI_KILL_*_TERMINATOR into exec-manipulating instructions. @@ -366,7 +383,6 @@ MDT = &getAnalysis(); SkipThreshold = SkipThresholdFlag; - MachineBasicBlock *EmptyMBBAtEnd = nullptr; SmallVector KillInstrs; bool MadeChange = false; @@ -438,6 +454,7 @@ MDT->getBase().insertEdge(&MBB, EmptyMBBAtEnd); } + break; default: @@ -452,6 +469,8 @@ Kill->eraseFromParent(); } KillInstrs.clear(); + EmptyMBBAtEnd = nullptr; + EarlyExitBlock = nullptr; return MadeChange; } diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir --- a/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir +++ b/llvm/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir @@ -14,13 +14,13 @@ # CHECK: bb.1: # CHECK: V_CMPX_LE_F32_e32 -# CHECK-NEXT: S_CBRANCH_EXECNZ %bb.2, implicit $exec +# CHECK-NEXT: S_CBRANCH_EXECZ %bb.3, implicit $exec -# CHECK: bb.3: -# CHECK-NEXT: EXP_DONE +# CHECK: bb.2: # CHECK: S_ENDPGM 0 -# CHECK: bb.2: +# CHECK: bb.3: +# CHECK-NEXT: EXP_DONE # CHECK: S_ENDPGM 0 name: kill_uncond_branch diff --git a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll --- a/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll +++ b/llvm/test/CodeGen/AMDGPU/skip-if-dead.ll @@ -12,11 +12,11 @@ ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: s_mov_b64 exec, 0 -; CHECK-NEXT: s_cbranch_execnz BB1_2 +; CHECK-NEXT: s_cbranch_execz BB1_2 ; CHECK-NEXT: ; %bb.1: -; CHECK-NEXT: exp null off, off, off, off done vm ; CHECK-NEXT: s_endpgm ; CHECK-NEXT: BB1_2: +; CHECK-NEXT: exp null off, off, off, off done vm ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_0_imm_neg() #0 { call void @llvm.amdgcn.kill(i1 false) @@ -27,15 +27,14 @@ ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg_x2: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: s_mov_b64 exec, 0 -; CHECK-NEXT: s_cbranch_execnz BB2_2 -; CHECK: exp null -; CHECK-NEXT: s_endpgm -; CHECK: BB2_2: +; CHECK-NEXT: s_cbranch_execz BB2_3 +; CHECK-NEXT: ; %bb.1: ; CHECK-NEXT: s_mov_b64 exec, 0 -; CHECK-NEXT: s_cbranch_execnz BB2_4 -; CHECK: exp null +; CHECK-NEXT: s_cbranch_execz BB2_3 +; CHECK-NEXT: ; %bb.2: ; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB2_4: +; CHECK-NEXT: BB2_3: +; CHECK: exp null ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_0_imm_neg_x2() #0 { call void @llvm.amdgcn.kill(i1 false) @@ -46,10 +45,11 @@ ; CHECK-LABEL: {{^}}test_kill_depth_var: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0 -; CHECK-NEXT: s_cbranch_execnz BB3_2 -; CHECK: exp null +; CHECK-NEXT: s_cbranch_execz BB3_2 +; CHECK-NEXT: ; %bb.1: ; CHECK-NEXT: s_endpgm ; CHECK-NEXT: BB3_2: +; CHECK: exp null ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_var(float %x) #0 { %cmp = fcmp olt float %x, 0.0 @@ -61,15 +61,14 @@ ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_same: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0 -; CHECK-NEXT: s_cbranch_execnz BB4_2 -; CHECK: exp null -; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB4_2: +; CHECK-NEXT: s_cbranch_execz BB4_3 +; CHECK-NEXT: ; %bb.1: ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0 -; CHECK-NEXT: s_cbranch_execnz BB4_4 -; CHECK: exp null +; CHECK-NEXT: s_cbranch_execz BB4_3 +; CHECK-NEXT: ; %bb.2: ; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB4_4: +; CHECK-NEXT: BB4_3: +; CHECK: exp null ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_var_x2_same(float %x) #0 { %cmp = fcmp olt float %x, 0.0 @@ -82,15 +81,14 @@ ; CHECK-LABEL: {{^}}test_kill_depth_var_x2: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0 -; CHECK-NEXT: s_cbranch_execnz BB5_2 -; CHECK: exp null -; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB5_2: +; CHECK-NEXT: s_cbranch_execz BB5_3 +; CHECK-NEXT: ; %bb.1 ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v1 -; CHECK-NEXT: s_cbranch_execnz BB5_4 -; CHECK: exp null +; CHECK-NEXT: s_cbranch_execz BB5_3 +; CHECK-NEXT: ; %bb.2 ; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB5_4: +; CHECK-NEXT: BB5_3: +; CHECK: exp null ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_var_x2(float %x, float %y) #0 { %cmp.x = fcmp olt float %x, 0.0 @@ -103,18 +101,15 @@ ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_instructions: ; CHECK-NEXT: ; %bb.0: ; CHECK-NEXT: v_cmpx_gt_f32_e32 vcc, 0, v0 -; CHECK-NEXT: s_cbranch_execnz BB6_2 +; CHECK-NEXT: s_cbranch_execz BB6_3 ; CHECK-NEXT: ; %bb.1: -; CHECK-NEXT: exp -; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB6_2: ; CHECK: v_mov_b32_e64 v7, -1 ; CHECK: v_cmpx_gt_f32_e32 vcc, 0, v7 -; CHECK-NEXT: s_cbranch_execnz BB6_4 -; CHECK-NEXT: ; %bb.3: -; CHECK-NEXT: exp +; CHECK-NEXT: s_cbranch_execz BB6_3 +; CHECK-NEXT: ; %bb.2: ; CHECK-NEXT: s_endpgm -; CHECK-NEXT: BB6_4: +; CHECK-NEXT: BB6_3: +; CHECK-NEXT: exp null ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 { %cmp.x = fcmp olt float %x, 0.0 @@ -295,12 +290,9 @@ ; CHECK-LABEL: {{^}}phi_use_def_before_kill: ; CHECK: v_cndmask_b32_e64 [[PHIREG:v[0-9]+]], 0, -1.0, ; CHECK: v_cmpx_lt_f32_e32 vcc, 0, -; CHECK-NEXT: s_cbranch_execnz [[BB4:BB[0-9]+_[0-9]+]] - -; CHECK: exp -; CHECK-NEXT: s_endpgm +; CHECK-NEXT: s_cbranch_execz [[EXITBB:BB[0-9]+_[0-9]+]] -; CHECK: [[KILLBB:BB[0-9]+_[0-9]+]]: +; CHECK: ; %[[KILLBB:bb.[0-9]+]]: ; CHECK-NEXT: s_cbranch_scc0 [[PHIBB:BB[0-9]+_[0-9]+]] ; CHECK: [[PHIBB]]: @@ -313,6 +305,10 @@ ; CHECK: [[ENDBB]]: ; CHECK-NEXT: s_endpgm + +; CHECK: [[EXITBB]]: +; CHECK: exp null +; CHECK-NEXT: s_endpgm define amdgpu_ps void @phi_use_def_before_kill(float inreg %x) #0 { bb: %tmp = fadd float %x, 1.000000e+00