AMDGPU/SI: Avoid moving PHIs to VALU when phi values are defined in scalar branches

[arsenm]

lib/Target/AMDGPU/SIFixSGPRCopies.cpp

101 ↗

(On Diff #67729)

Should also preserve it (it might be redundant with preserves CFG but I'm not sure if you actually use it)

248 ↗

(On Diff #67729)

Reference

249–266 ↗

(On Diff #67729)

This is also missing s_cbranch_exec[n]z, s_endpgm, si_return.

This function seems generally wrong to me. What exactly are you looking for by uniform terminator? The way I've been thinking about it, any branch terminator is uniform.

This only looks at a single terminator, so it won't correctly handle something like:

s_cbranch_scc0 BB1
s_branch BB2

I think what you really want is to look for is si_mask_branch (or at this early point one of the pseudo terminators that modify exec, none of which should be branches).

But you can also have something like:
si_mask_branch BB1
s_cbranch_vccz BB1
s_branch BB2

Coming directly out of isel I think this is the most likely scenario since BranchFolding is supposed to cleanup unneeded unconditional branches. I think what you probably want to be looking for is really a terminator that modifiers exec.

307 ↗

(On Diff #67729)

s/denominator/dominator

[• tstellarAMD]

Address review comments.

[arsenm]

LGTM

llvm/trunk/lib/Target/AMDGPU/SIFixSGPRCopies.cpp

 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
 /// will be unable to perform the COPY removal from the above example  which
 /// ultimately led to the creation of an illegal COPY.
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "AMDGPUSubtarget.h"
 #include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "si-fix-sgpr-copies"
 
 namespace {
 
 class SIFixSGPRCopies : public MachineFunctionPass {
+
+  MachineDominatorTree *MDT;
+
 public:
   static char ID;
 
   SIFixSGPRCopies() : MachineFunctionPass(ID) { }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   StringRef getPassName() const override { return "SI Fix SGPR copies"; }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<MachineDominatorTree>();
+    AU.addPreserved<MachineDominatorTree>();
     AU.setPreservesCFG();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
 
 } // End anonymous namespace
 
-INITIALIZE_PASS(SIFixSGPRCopies, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
                      "SI Fix SGPR copies", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree)
+INITIALIZE_PASS_END(SIFixSGPRCopies, DEBUG_TYPE,
+                     "SI Fix SGPR copies", false, false)
+
 
 char SIFixSGPRCopies::ID = 0;
 
 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
 
 FunctionPass *llvm::createSIFixSGPRCopiesPass() {
   return new SIFixSGPRCopies();
 }
for (unsigned i = 1; i < PHI.getNumOperands(); i += 2) {
     case AMDGPU::PHI:
       if (phiHasBreakDef(*DefInstr, MRI, Visited))
         return true;
     }
   }
   return false;
 }
 
+static bool hasTerminatorThatModifiesExec(const MachineBasicBlock &MBB,
+                                          const TargetRegisterInfo &TRI) {
+  for (MachineBasicBlock::const_iterator I = MBB.getFirstTerminator(),
+       E = MBB.end(); I != E; ++I) {
+    if (I->modifiesRegister(AMDGPU::EXEC, &TRI))
+      return true;
+  }
+  return false;
+}
+
 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
   const SIInstrInfo *TII = ST.getInstrInfo();
+  MDT = &getAnalysis<MachineDominatorTree>();
 
   SmallVector<MachineInstr *, 16> Worklist;
 
   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
                                                   BI != BE; ++BI) {
 
     MachineBasicBlock &MBB = *BI;
     for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
         if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) {
           DEBUG(dbgs() << "Fixing VGPR -> SGPR copy: " << MI);
           TII->moveToVALU(MI);
         }
 
         break;
       }
       case AMDGPU::PHI: {
-        DEBUG(dbgs() << "Fixing PHI: " << MI);
         unsigned Reg = MI.getOperand(0).getReg();
         if (!TRI->isSGPRClass(MRI.getRegClass(Reg)))
           break;
 
+        // We don't need to fix the PHI if the common dominator of the
+        // two incoming blocks terminates with a uniform branch.
+        if (MI.getNumExplicitOperands() == 5) {
+          MachineBasicBlock *MBB0 = MI.getOperand(2).getMBB();
+          MachineBasicBlock *MBB1 = MI.getOperand(4).getMBB();
+
+          MachineBasicBlock *NCD = MDT->findNearestCommonDominator(MBB0, MBB1);
+          if (NCD && !hasTerminatorThatModifiesExec(*NCD, *TRI)) {
+            DEBUG(dbgs() << "Not fixing PHI for uniform branch: " << MI << '\n');
+            break;
+          }
+        }
+
         // If a PHI node defines an SGPR and any of its operands are VGPRs,
         // then we need to move it to the VALU.
         //
         // Also, if a PHI node defines an SGPR and has all SGPR operands
         // we must move it to the VALU, because the SGPR operands will
         // all end up being assigned the same register, which means
         // there is a potential for a conflict if different threads take
         // different control flow paths.
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
         // Will Become:
         //
         // sgpr2 = def;
         // ...
         // sgpr2 = def;
         // ...
         // use sgpr2
         //
-        // FIXME: This is OK if the branching decision is made based on an
-        // SGPR value.
-        bool SGPRBranch = false;
-
         // The one exception to this rule is when one of the operands
         // is defined by a SI_BREAK, SI_IF_BREAK, or SI_ELSE_BREAK
         // instruction.  In this case, there we know the program will
         // never enter the second block (the loop) without entering
         // the first block (where the condition is computed), so there
         // is no chance for values to be over-written.
 
         SmallSet<unsigned, 8> Visited;
         if (phiHasVGPROperands(MI, MRI, TRI, TII) ||
-            (!SGPRBranch && !phiHasBreakDef(MI, MRI, Visited))) {
+            !phiHasBreakDef(MI, MRI, Visited)) {
+          DEBUG(dbgs() << "Fixing PHI: " << MI);
           TII->moveToVALU(MI);
         }
         break;
       }
       case AMDGPU::REG_SEQUENCE: {
         if (TRI->hasVGPRs(TII->getOpRegClass(MI, 0)) ||
             !hasVGPROperands(MI, TRI)) {
           foldVGPRCopyIntoRegSequence(MI, TRI, TII, MRI);

llvm/trunk/test/CodeGen/AMDGPU/branch-relaxation.ll

call void asm sideeffect " ; 32 bytes
     v_nop_e64", ""() #0
   br label %bb3
 
 bb3:
   store volatile i32 %load, i32 addrspace(1)* %gep
   ret void
 }
 
-; FIXME: Should be able to use s_cbranch_scc0
 ; GCN-LABEL: {{^}}long_backward_sbranch:
-; GCN: v_mov_b32_e32 [[LOOPIDX:v[0-9]+]], 0{{$}}
+; GCN: s_mov_b32 [[LOOPIDX:s[0-9]+]], 0{{$}}
 
 ; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: ; %bb2
 ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_add_i32_e32 [[INC:v[0-9]+]], vcc, 1, [[LOOPIDX]]
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 10, [[INC]]
-; GCN-NEXT: s_and_b64 vcc, exec, vcc
+; GCN-NEXT: s_add_i32 [[INC:s[0-9]+]], [[LOOPIDX]], 1
+; GCN-NEXT: s_cmp_lt_i32 [[INC]], 10
 
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: ;;#ASMEND
 
-; GCN-NEXT: s_cbranch_vccz [[ENDBB:BB[0-9]+_[0-9]+]]
+; GCN-NEXT: s_cbranch_scc0 [[ENDBB:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT: [[LONG_JUMP:BB[0-9]+_[0-9]+]]: ; %bb2
 ; GCN-NEXT: ; in Loop: Header=[[LOOPBB]] Depth=1
 ; GCN-NEXT: s_getpc_b64 vcc
 ; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONG_JUMP]]+4)-[[LOOPBB]]
 ; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0
 ; GCN-NEXT: s_setpc_b64 vcc
 

llvm/trunk/test/CodeGen/AMDGPU/cf-loop-on-constant.ll

for.body:
   br i1 undef, label %for.body, label %for.exit
 }
 
 ; GCN-LABEL: {{^}}loop_arg_0:
 ; GCN: v_and_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}}
 ; GCN: v_cmp_eq_u32_e32 vcc, 1,
 
 ; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80
+; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 4
+
 ; GCN: s_cbranch_vccnz [[LOOPBB]]
 ; GCN-NEXT: ; BB#2
 ; GCN-NEXT: s_endpgm
 define void @loop_arg_0(float addrspace(3)* %ptr, i32 %n, i1 %cond) nounwind {
 entry:
   br label %for.body
 
 for.exit:

llvm/trunk/test/CodeGen/AMDGPU/coalescer_remat.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs -mtriple=amdgcn-- -o - %s | FileCheck %s
 
 declare float @llvm.fma.f32(float, float, float)
 
 ; This checks that rematerialization support of the coalescer does not
 ; unnecessarily widen the register class. Without those fixes > 20 VGprs
 ; are used here
 ; Also check that some rematerialization of the 0 constant happened.
 ; CHECK-LABEL: foobar
 ; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
 ; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
 ; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
 ; CHECK:  v_mov_b32_e32 v{{[0-9]+}}, 0
 ; It's probably OK if this is slightly higher:
-; CHECK: ; NumVgprs: 9
+; CHECK: ; NumVgprs: 8
 define void @foobar(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in, i32 %flag) {
 entry:
   %cmpflag = icmp eq i32 %flag, 1
   br i1 %cmpflag, label %loop, label %exit
 
 loop:
   %c = phi i32 [0, %entry], [%cnext, %loop]
   %v0 = phi float [0.0, %entry], [%fma.0, %loop]

llvm/trunk/test/CodeGen/AMDGPU/hoist-cond.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
 
 ; Check that invariant compare is hoisted out of the loop.
 ; At the same time condition shall not be serialized into a VGPR and deserialized later
 ; using another v_cmp + v_cndmask, but used directly in s_and_saveexec_b64.
 
-; CHECK: v_cmp_{{..}}_u32_e64 [[COND:s\[[0-9]+:[0-9]+\]]]
+; CHECK: v_cmp_{{..}}_u32_e{{32|64}} [[COND:s\[[0-9]+:[0-9]+\]|vcc]]
 ; CHECK: BB0_1:
 ; CHECK-NOT: v_cmp
 ; CHECK_NOT: v_cndmask
 ; CHECK: s_and_saveexec_b64 s[{{[[0-9]+:[0-9]+}}], [[COND]]
 ; CHECK: BB0_2:
 
 define amdgpu_kernel void @hoist_cond(float addrspace(1)* nocapture %arg, float addrspace(1)* noalias nocapture readonly %arg1, i32 %arg3, i32 %arg4) {
 bb:

llvm/trunk/test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
+; FIXME: Enabling critical edge splitting will fix this.
+; XFAIL: *
+
 ; Make sure that m0 is not reinitialized in the loop.
 
 ; GCN-LABEL: {{^}}copy_local_to_global_loop_m0_init:
 ; GCN: s_cbranch_scc1 BB0_3
 
 ; Initialize in preheader
 ; GCN: s_mov_b32 m0, -1
 
 ; GCN: BB0_2:
 ; GCN: ds_read_b32
 ; GCN: buffer_store_dword
 
-; GCN: s_cbranch_vccz BB0_2
+; GCN: s_cbranch_scc0 BB0_2
 
 ; GCN: BB0_3:
 ; GCN-NEXT: s_endpgm
 define void @copy_local_to_global_loop_m0_init(i32 addrspace(1)* noalias nocapture %out, i32 addrspace(3)* noalias nocapture readonly %in, i32 %n) #0 {
 bb:
   %tmp = icmp sgt i32 %n, 0
   br i1 %tmp, label %.lr.ph.preheader, label %._crit_edge
 

llvm/trunk/test/CodeGen/AMDGPU/loop_break.ll

 ; OPT: call void @llvm.amdgcn.end.cf(i64
 
 ; TODO: Can remove exec fixes in return block
 ; GCN-LABEL: {{^}}break_loop:
 ; GCN: s_mov_b64 [[INITMASK:s\[[0-9]+:[0-9]+\]]], 0{{$}}
 
 ; GCN: [[LOOP_ENTRY:BB[0-9]+_[0-9]+]]: ; %bb1
 ; GCN: s_or_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[INITMASK]]
-; GCN: v_cmp_lt_i32_e32 vcc,
-; GCN: s_and_b64 vcc, exec, vcc
-; GCN-NEXT: s_cbranch_vccnz [[FLOW:BB[0-9]+_[0-9]+]]
+; GCN: s_cmp_gt_i32 s{{[0-9]+}}, -1
+; GCN-NEXT: s_cbranch_scc1 [[FLOW:BB[0-9]+_[0-9]+]]
 
 ; GCN: ; BB#2: ; %bb4
 ; GCN: buffer_load_dword
 ; GCN: v_cmp_ge_i32_e32 vcc,
 ; GCN: s_or_b64 [[MASK]], vcc, [[INITMASK]]
 
 ; GCN: [[FLOW]]:
 ; GCN: s_mov_b64 [[INITMASK]], [[MASK]]

llvm/trunk/test/CodeGen/AMDGPU/uniform-cfg.ll

-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=verde -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}uniform_if_scc:
 ; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 ; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_scc(i32 %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %if, label %else
 
 if:
   br label %done
 
 else:
   br label %done
 
 done:
   %value = phi i32 [0, %if], [1, %else]
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_vcc:
-; FIXME: We could use _e32 here if we re-used the 0 from [[STORE_VAL]], and
-; also scheduled the write first.
 ; GCN-DAG: v_cmp_eq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}}
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 ; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_vcc(float %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = fcmp oeq float %cond, 0.0
   br i1 %cmp0, label %if, label %else
 
 if:
   br label %done
 
 else:
   br label %done
 
 done:
   %value = phi i32 [0, %if], [1, %else]
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_swap_br_targets_scc:
 ; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 ; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_swap_br_targets_scc(i32 %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = icmp eq i32 %cond, 0
   br i1 %cmp0, label %else, label %if
 
 if:
   br label %done
 
 else:
   br label %done
 
 done:
   %value = phi i32 [0, %if], [1, %else]
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_swap_br_targets_vcc:
-; FIXME: We could use _e32 here if we re-used the 0 from [[STORE_VAL]], and
-; also scheduled the write first.
 ; GCN-DAG: v_cmp_neq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}}
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 ; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_swap_br_targets_vcc(float %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = fcmp oeq float %cond, 0.0
   br i1 %cmp0, label %else, label %if
 
 if:
   br label %done
 

 bb7:                                              ; preds = %bb5
   store i32 %tmp3, i32 addrspace(1)* %out
   br label %bb9
 
 bb9:                                              ; preds = %bb8, %bb4
   ret void
 }
 
-; GCN-LABEL: {{^}}uniform_loop:
-; GCN: {{^}}[[LOOP_LABEL:[A-Z0-9_a-z]+]]:
-; FIXME: We need to teach GCNFixSGPRCopies about uniform branches so we
-;        get s_add_i32 here.
-; GCN: v_add_i32_e32 [[I:v[0-9]+]], vcc, -1, v{{[0-9]+}}
-; GCN: v_cmp_ne_u32_e32 vcc, 0, [[I]]
-; GCN: s_and_b64 vcc, exec, vcc
-; GCN: s_cbranch_vccnz [[LOOP_LABEL]]
-; GCN: s_endpgm
+; SI-LABEL: {{^}}uniform_loop:
+; SI: {{^}}[[LOOP_LABEL:[A-Z0-9_a-z]+]]:
+; SI: s_add_i32 [[I:s[0-9]+]],  s{{[0-9]+}}, -1
+; SI: s_cmp_lg_u32 [[I]], 0
+; SI: s_cbranch_scc1 [[LOOP_LABEL]]
+; SI: s_endpgm
 define void @uniform_loop(i32 addrspace(1)* %out, i32 %a) {
 entry:
   br label %loop
 
 loop:
   %i = phi i32 [0, %entry], [%i.i, %loop]
   %i.i = add i32 %i, 1
   %cmp = icmp eq i32 %a, %i.i
bb7:                                              ; preds = %bb5
   br label %bb9
 
 bb9:                                              ; preds = %bb8, %bb4
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_scc_i64_eq:
 ; VI-DAG: s_cmp_eq_u64 s{{\[[0-9]+:[0-9]+\]}}, 0
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 
 ; SI: v_cmp_eq_u64_e64
 ; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_scc_i64_eq(i64 %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = icmp eq i64 %cond, 0
   br i1 %cmp0, label %if, label %else
 
 if:
   br label %done
 
 else:
   br label %done
 
 done:
   %value = phi i32 [0, %if], [1, %else]
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_scc_i64_ne:
 ; VI-DAG: s_cmp_lg_u64 s{{\[[0-9]+:[0-9]+\]}}, 0
-; GCN-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0
+; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 
 ; SI: v_cmp_ne_u64_e64
 ; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_scc_i64_ne(i64 %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = icmp ne i64 %cond, 0
   br i1 %cmp0, label %if, label %else
 
 if:
   br label %done
 
 else:
   br label %done
 
 done:
   %value = phi i32 [0, %if], [1, %else]
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}uniform_if_scc_i64_sgt:
+; GCN: s_mov_b32 [[S_VAL:s[0-9]+]], 0
 ; GCN: v_cmp_gt_i64_e64
 ; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]]
 
 ; Fall-through to the else
-; GCN: v_mov_b32_e32 [[STORE_VAL]], 1
+; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: buffer_store_dword [[STORE_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL]], [[S_VAL]]
+; GCN: buffer_store_dword [[V_VAL]]
 define void @uniform_if_scc_i64_sgt(i64 %cond, i32 addrspace(1)* %out) {
 entry:
   %cmp0 = icmp sgt i64 %cond, 0
   br i1 %cmp0, label %if, label %else
 
 if:
   br label %done
 

llvm/trunk/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll

 ; RUN: llc -march=amdgcn -mcpu=verde < %s | FileCheck %s
 
 ; Test a simple uniform loop that lives inside non-uniform control flow.
 
 ; CHECK-LABEL: {{^}}test1:
 ; CHECK: v_cmp_ne_u32_e32 vcc, 0
 ; CHECK: s_and_saveexec_b64
 ; CHECK-NEXT: s_xor_b64
 ; CHECK-NEXT: ; mask branch
-; CHECK-NEXT: s_cbranch_execz
 
 ; CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %loop_body.preheader
 
 ; CHECK: [[LOOP_BODY_LABEL:BB[0-9]+_[0-9]+]]:
-; CHECK: s_and_b64 vcc, exec, vcc
-; CHECK: s_cbranch_vccz [[LOOP_BODY_LABEL]]
+; CHECK: s_cbranch_scc0 [[LOOP_BODY_LABEL]]
 
 ; CHECK: s_endpgm
 define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <2 x i32> %addr.base, i32 %y, i32 %p) {
 main_body:
   %cc = icmp eq i32 %p, 0
   br i1 %cc, label %out, label %loop_body
 
 loop_body:

llvm/trunk/test/CodeGen/AMDGPU/valu-i1.ll

 ; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
 ; SI: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]]
 
 ; SI: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0{{$}}
 
 ; SI: [[LABEL_LOOP:BB[0-9]+_[0-9]+]]:
 ; SI: buffer_load_dword
 ; SI-DAG: buffer_store_dword
-; SI-DAG: v_cmp_eq_u32_e32 vcc,
-; SI-DAG: s_and_b64 vcc, exec, vcc
-; SI: s_cbranch_vccz [[LABEL_LOOP]]
+; SI-DAG: s_cmpk_eq_i32 s{{[0-9]+}}, 0x100
+; SI: s_cbranch_scc0 [[LABEL_LOOP]]
 ; SI: [[LABEL_EXIT]]:
 ; SI: s_endpgm
 
 define void @simple_test_v_loop(i32 addrspace(1)* %dst, i32 addrspace(1)* %src) #1 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %is.0 = icmp ne i32 %tid, 0
   %limit = add i32 %tid, 64