[RFC] AMDGPU: add execfix flag to SI_ELSE

[nhaehnle]

Summary

SI_ELSE is lowered into two parts:

s_or_saveexec_b64 dst, src (at the start of the basic block)

s_xor_b64 exec, exec, dst (at the end of the basic block)

The idea is that dst contains the exec mask of the preceding IF block. It can
happen that SIWholeQuadMode decides to switch from WQM to Exact mode inside
the basic block that contains SI_ELSE, in which case it introduces an instruction

s_and_b64 exec, exec, s[...]

which masks out bits that can correspond to both the IF and the ELSE paths.
So the resulting sequence must be:

s_or_savexec_b64 dst, src

s_and_b64 exec, exec, s[...] <-- added by SIWholeQuadMode
s_and_b64 dst, dst, exec <-- added by SILowerControlFlow

s_xor_b64 exec, exec, dst

Whether to add the additional s_and_b64 dst, dst, exec is currently determined
via the ExecModified tracking. With this change, it is instead determined by
an additional flag on SI_ELSE which is set by SIWholeQuadMode.

Finally: It also occured to me that an alternative approach for the long run
is for SILowerControlFlow to unconditionally emit

s_or_saveexec_b64 dst, src

...

s_and_b64 dst, dst, exec
s_xor_b64 exec, exec, dst

and have a pass that detects and cleans up the "redundant AND with exec"
pattern where possible. This could be useful anyway, because we also add
instructions

s_and_b64 vcc, exec, vcc

before s_cbranch_scc (in moveToALU), and those are often redundant. I have
some pending changes to how KILL is lowered that could also benefit from
such a cleanup pass.

In any case, this current patch could help in the short term with the whole
ExecModified business.

[arsenm]

LGTM, except for the operand type. Also, can you add a test which shows a change from moving the modifiesRegister so that it doesn't also count the else (since I think this should fix it?)

lib/Target/AMDGPU/SIInstructions.td

1954 ↗

(On Diff #65671)

This should be i1imm instead of SSrc since it's just an immediate

llvm/trunk/lib/Target/AMDGPU/SIInstructions.td

 
 def SI_IF: PseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$vcc, brtarget:$target),
   [(set i64:$dst, (int_amdgcn_if i1:$vcc, bb:$target))]> {
   let Constraints = "";
 }
 
 def SI_ELSE : PseudoInstSI <
-  (outs SReg_64:$dst), (ins SReg_64:$src, brtarget:$target),
-  [(set i64:$dst, (int_amdgcn_else i64:$src, bb:$target))]> {
+  (outs SReg_64:$dst), (ins SReg_64:$src, brtarget:$target, i1imm:$execfix)> {
   let Constraints = "$src = $dst";
 }
 
 def SI_LOOP : PseudoInstSI <
   (outs), (ins SReg_64:$saved, brtarget:$target),
   [(int_amdgcn_loop i64:$saved, bb:$target)]
 >;
 

 }
 
 } // End Defs = [SCC]
 
 } // End SubtargetPredicate = isGCN
 
 let Predicates = [isGCN] in {
 
-def : Pat <
+def : Pat<
+  (int_amdgcn_else i64:$src, bb:$target),
+  (SI_ELSE $src, $target, 0)
+>;
+
+def : Pat <
   (int_AMDGPU_kilp),
   (SI_KILL 0xbf800000)
 >;
 
 /* int_SI_vs_load_input */
 def : Pat<
   (SIload_input v4i32:$tlst, imm:$attr_offset, i32:$buf_idx_vgpr),
   (BUFFER_LOAD_FORMAT_XYZW_IDXEN $buf_idx_vgpr, $tlst, 0, imm:$attr_offset, 0, 0, 0)

llvm/trunk/lib/Target/AMDGPU/SILowerControlFlow.cpp

private:
   unsigned SkipThreshold;
 
   bool shouldSkip(MachineBasicBlock *From, MachineBasicBlock *To);
 
   void Skip(MachineInstr &From, MachineOperand &To);
   bool skipIfDead(MachineInstr &MI, MachineBasicBlock &NextBB);
 
   void If(MachineInstr &MI);
-  void Else(MachineInstr &MI, bool ExecModified);
+  void Else(MachineInstr &MI);
   void Break(MachineInstr &MI);
   void IfBreak(MachineInstr &MI);
   void ElseBreak(MachineInstr &MI);
   void Loop(MachineInstr &MI);
   void EndCf(MachineInstr &MI);
 
   void Kill(MachineInstr &MI);
   void Branch(MachineInstr &MI);
void SILowerControlFlow::If(MachineInstr &MI) {
   // Insert a pseudo terminator to help keep the verifier happy.
   BuildMI(MBB, &MI, DL, TII->get(AMDGPU::SI_MASK_BRANCH))
     .addOperand(MI.getOperand(2))
     .addReg(Reg);
 
   MI.eraseFromParent();
 }
 
-void SILowerControlFlow::Else(MachineInstr &MI, bool ExecModified) {
+void SILowerControlFlow::Else(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc DL = MI.getDebugLoc();
   unsigned Dst = MI.getOperand(0).getReg();
   unsigned Src = MI.getOperand(1).getReg();
 
   BuildMI(MBB, MBB.getFirstNonPHI(), DL,
           TII->get(AMDGPU::S_OR_SAVEEXEC_B64), Dst)
           .addReg(Src); // Saved EXEC
 
-  if (ExecModified) {
+  if (MI.getOperand(3).getImm() != 0) {
     // Adjust the saved exec to account for the modifications during the flow
     // block that contains the ELSE. This can happen when WQM mode is switched
     // off.
     BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_AND_B64), Dst)
             .addReg(AMDGPU::EXEC)
             .addReg(Dst);
   }
 
bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
 
   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
        BI != BE; BI = NextBB) {
     NextBB = std::next(BI);
     MachineBasicBlock &MBB = *BI;
 
     MachineBasicBlock *EmptyMBBAtEnd = nullptr;
     MachineBasicBlock::iterator I, Next;
-    bool ExecModified = false;
 
     for (I = MBB.begin(); I != MBB.end(); I = Next) {
       Next = std::next(I);
 
       MachineInstr &MI = *I;
 
       // Flat uses m0 in case it needs to access LDS.
       if (TII->isFLAT(MI))
         NeedFlat = true;
 
-      if (I->modifiesRegister(AMDGPU::EXEC, TRI))
-        ExecModified = true;
-
       switch (MI.getOpcode()) {
         default: break;
         case AMDGPU::SI_IF:
           ++Depth;
           If(MI);
           break;
 
         case AMDGPU::SI_ELSE:
-          Else(MI, ExecModified);
+          Else(MI);
           break;
 
         case AMDGPU::SI_BREAK:
           Break(MI);
           break;
 
         case AMDGPU::SI_IF_BREAK:
           IfBreak(MI);

llvm/trunk/lib/Target/AMDGPU/SIWholeQuadMode.cpp

if (Needs && State != Needs) {
       } else {
         assert(WQMFromExec == (SavedWQMReg == 0));
         toWQM(MBB, &MI, SavedWQMReg);
         SavedWQMReg = 0;
       }
 
       State = Needs;
     }
+
+    if (MI.getOpcode() == AMDGPU::SI_ELSE && State == StateExact)
+      MI.getOperand(3).setImm(1);
   }
 
   if ((BI.OutNeeds & StateWQM) && State != StateWQM) {
     assert(WQMFromExec == (SavedWQMReg == 0));
     toWQM(MBB, MBB.end(), SavedWQMReg);
   } else if (BI.OutNeeds == StateExact && State != StateExact) {
     toExact(MBB, MBB.end(), 0, LiveMaskReg);
   }

llvm/trunk/test/CodeGen/AMDGPU/else.ll

+; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
+
+; CHECK-LABEL: {{^}}else_no_execfix:
+; CHECK: ; %Flow
+; CHECK-NEXT: s_or_saveexec_b64 [[DST:s\[[0-9]+:[0-9]+\]]],
+; CHECK-NEXT: s_xor_b64 exec, exec, [[DST]]
+; CHECK-NEXT: ; mask branch
+define amdgpu_ps float @else_no_execfix(i32 %z, float %v) {
+main_body:
+  %cc = icmp sgt i32 %z, 5
+  br i1 %cc, label %if, label %else
+
+if:
+  %v.if = fmul float %v, 2.0
+  br label %end
+
+else:
+  %v.else = fmul float %v, 3.0
+  br label %end
+
+end:
+  %r = phi float [ %v.if, %if ], [ %v.else, %else ]
+  ret float %r
+}
+
+; CHECK-LABEL: {{^}}else_execfix_leave_wqm:
+; CHECK: ; %Flow
+; CHECK-NEXT: s_or_saveexec_b64 [[DST:s\[[0-9]+:[0-9]+\]]],
+; CHECK-NEXT: s_and_b64 exec, exec,
+; CHECK-NEXT: s_and_b64 [[DST]], exec, [[DST]]
+; CHECK-NEXT: s_xor_b64 exec, exec, [[DST]]
+; CHECK-NEXT: ; mask branch
+define amdgpu_ps void @else_execfix_leave_wqm(i32 %z, float %v) {
+main_body:
+  %cc = icmp sgt i32 %z, 5
+  br i1 %cc, label %if, label %else
+
+if:
+  %v.if = fmul float %v, 2.0
+  br label %end
+
+else:
+  %c = fmul float %v, 3.0
+  %c.i = bitcast float %c to i32
+  %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c.i, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
+  %v.else = extractelement <4 x float> %tex, i32 0
+  br label %end
+
+end:
+  %r = phi float [ %v.if, %if ], [ %v.else, %else ]
+  call void @llvm.amdgcn.buffer.store.f32(float %r, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) nounwind
+
+declare <4 x float> @llvm.SI.image.sample.i32(i32, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) nounwind readnone