ScheduleDAGInstrs: Add condjump deps in addSchedBarrierDeps()

[MatzeB]

Summary

addSchedBarrierDeps() is supposed to add use operands to the ExitSU
node. The current implementation adds uses for calls/barrier instruction
and the MBB live-outs in all other cases. The use
operands of conditional jump instructions were missed.

(I'd also find adding the live-outs of the basic block questionable as I would assume it to be more likely a value is not used immediately, but we can have that discussion another day)

[MatzeB]

Note that this is a review to get some feedback on the change, testcase adjustments are still outstanding.

[kparzysz]

Looks ok to me.

Should def operands in branches also be treated specially (e.g. adding a dependency with a successor's live-in)?

[MatzeB]

In D25140#559480, @kparzysz wrote:

Should def operands in branches also be treated specially (e.g. adding a dependency with a successor's live-in)?

It shouldn't matter for scheduling: Branch instructions are only allowed at the end of a basic block, so by definition we cannot have any non-branch instructions between them, so we will never get any interesting scheduling region formed between branches.

[MatzeB]

New version:

• Adjust test cases to changed schedule
• Now that we have proper dependency on the conditional branches we can simplify the macro fusion code to only check the ExitSU predecessors and we can also skip the dependency check there.

[• tstellarAMD]

AMDGPU changes look OK to me.

[MatzeB]

ping

[atrick]

LGTM.
Thanks!

llvm/trunk/include/llvm/CodeGen/MachineScheduler.h

 createLoadClusterDAGMutation(const TargetInstrInfo *TII,
                              const TargetRegisterInfo *TRI);
 
 std::unique_ptr<ScheduleDAGMutation>
 createStoreClusterDAGMutation(const TargetInstrInfo *TII,
                               const TargetRegisterInfo *TRI);
 
 std::unique_ptr<ScheduleDAGMutation>
-createMacroFusionDAGMutation(const TargetInstrInfo *TII,
-                             const TargetRegisterInfo *TRI);
+createMacroFusionDAGMutation(const TargetInstrInfo *TII);
 
 std::unique_ptr<ScheduleDAGMutation>
 createCopyConstrainDAGMutation(const TargetInstrInfo *TII,
                                const TargetRegisterInfo *TRI);
 
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_MACHINESCHEDULER_H

llvm/trunk/lib/CodeGen/MachineScheduler.cpp

 // MacroFusion - DAG post-processing to encourage fusion of macro ops.
 //===----------------------------------------------------------------------===//
 
 namespace {
 /// \brief Post-process the DAG to create cluster edges between instructions
 /// that may be fused by the processor into a single operation.
 class MacroFusion : public ScheduleDAGMutation {
   const TargetInstrInfo &TII;
-  const TargetRegisterInfo &TRI;
 public:
-  MacroFusion(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)
-    : TII(TII), TRI(TRI) {}
+  MacroFusion(const TargetInstrInfo &TII)
+    : TII(TII) {}
 
   void apply(ScheduleDAGInstrs *DAGInstrs) override;
 };
 } // anonymous
 
 namespace llvm {
 
 std::unique_ptr<ScheduleDAGMutation>
-createMacroFusionDAGMutation(const TargetInstrInfo *TII,
-                             const TargetRegisterInfo *TRI) {
-  return make_unique<MacroFusion>(*TII, *TRI);
+createMacroFusionDAGMutation(const TargetInstrInfo *TII) {
+  return make_unique<MacroFusion>(*TII);
 }
 
 } // namespace llvm
 
-/// Returns true if \p MI reads a register written by \p Other.
-static bool HasDataDep(const TargetRegisterInfo &TRI, const MachineInstr &MI,
-                       const MachineInstr &Other) {
-  for (const MachineOperand &MO : MI.uses()) {
-    if (!MO.isReg() || !MO.readsReg())
-      continue;
-
-    unsigned Reg = MO.getReg();
-    if (Other.modifiesRegister(Reg, &TRI))
-      return true;
-  }
-  return false;
-}
-
 /// \brief Callback from DAG postProcessing to create cluster edges to encourage
 /// fused operations.
 void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) {
   ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
 
   // For now, assume targets can only fuse with the branch.
   SUnit &ExitSU = DAG->ExitSU;
   MachineInstr *Branch = ExitSU.getInstr();
   if (!Branch)
     return;
 
-  for (SUnit &SU : DAG->SUnits) {
-    // SUnits with successors can't be schedule in front of the ExitSU.
-    if (!SU.Succs.empty())
-      continue;
-    // We only care if the node writes to a register that the branch reads.
-    MachineInstr *Pred = SU.getInstr();
-    if (!HasDataDep(TRI, *Branch, *Pred))
+  for (SDep &PredDep : ExitSU.Preds) {
+    if (PredDep.isWeak())
       continue;
-
-    if (!TII.shouldScheduleAdjacent(*Pred, *Branch))
+    SUnit &SU = *PredDep.getSUnit();
+    MachineInstr &Pred = *SU.getInstr();
+    if (!TII.shouldScheduleAdjacent(Pred, *Branch))
       continue;
 
     // Create a single weak edge from SU to ExitSU. The only effect is to cause
     // bottom-up scheduling to heavily prioritize the clustered SU.  There is no
     // need to copy predecessor edges from ExitSU to SU, since top-down
     // scheduling cannot prioritize ExitSU anyway. To defer top-down scheduling
     // of SU, we could create an artificial edge from the deepest root, but it
     // hasn't been needed yet.
     bool Success = DAG->addEdge(&ExitSU, SDep(&SU, SDep::Cluster));
     (void)Success;
     assert(Success && "No DAG nodes should be reachable from ExitSU");
 
+    // Adjust latency of data deps between the nodes.
+    for (SDep &PredDep : ExitSU.Preds) {
+      if (PredDep.getSUnit() == &SU)
+        PredDep.setLatency(0);
+    }
+    for (SDep &SuccDep : SU.Succs) {
+      if (SuccDep.getSUnit() == &ExitSU)
+        SuccDep.setLatency(0);
+    }
+
     DEBUG(dbgs() << "Macro Fuse SU(" << SU.NodeNum << ")\n");
     break;
   }
 }
 
 //===----------------------------------------------------------------------===//
 // CopyConstrain - DAG post-processing to encourage copy elimination.
 //===----------------------------------------------------------------------===//
static ScheduleDAGInstrs *createGenericSchedLive(MachineSchedContext *C) {
   DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI));
   if (EnableMemOpCluster) {
     if (DAG->TII->enableClusterLoads())
       DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
     if (DAG->TII->enableClusterStores())
       DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
   }
   if (EnableMacroFusion)
-    DAG->addMutation(createMacroFusionDAGMutation(DAG->TII, DAG->TRI));
+    DAG->addMutation(createMacroFusionDAGMutation(DAG->TII));
   return DAG;
 }
 
 static MachineSchedRegistry
 GenericSchedRegistry("converge", "Standard converging scheduler.",
                      createGenericSchedLive);
 
 //===----------------------------------------------------------------------===//

llvm/trunk/lib/CodeGen/ScheduleDAGInstrs.cpp

 /// make sure instructions which define registers that are either used by
 /// the terminator or are live-out are properly scheduled. This is
 /// especially important when the definition latency of the return value(s)
 /// are too high to be hidden by the branch or when the liveout registers
 /// used by instructions in the fallthrough block.
 void ScheduleDAGInstrs::addSchedBarrierDeps() {
   MachineInstr *ExitMI = RegionEnd != BB->end() ? &*RegionEnd : nullptr;
   ExitSU.setInstr(ExitMI);
-  bool AllDepKnown = ExitMI &&
-    (ExitMI->isCall() || ExitMI->isBarrier());
-  if (ExitMI && AllDepKnown) {
-    // If it's a call or a barrier, add dependencies on the defs and uses of
-    // instruction.
+  // Add dependencies on the defs and uses of the instruction.
+  if (ExitMI) {
     for (const MachineOperand &MO : ExitMI->operands()) {
       if (!MO.isReg() || MO.isDef()) continue;
       unsigned Reg = MO.getReg();
       if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
         Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg));
       } else if (TargetRegisterInfo::isVirtualRegister(Reg) && MO.readsReg()) {
         addVRegUseDeps(&ExitSU, ExitMI->getOperandNo(&MO));
       }
     }
-  } else {
+  }
+  if (!ExitMI || (!ExitMI->isCall() && !ExitMI->isBarrier())) {
     // For others, e.g. fallthrough, conditional branch, assume the exit
     // uses all the registers that are livein to the successor blocks.
-    assert(Uses.empty() && "Uses in set before adding deps?");
     for (const MachineBasicBlock *Succ : BB->successors()) {
       for (const auto &LI : Succ->liveins()) {
         if (!Uses.contains(LI.PhysReg))
           Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg));
       }
     }
   }
 }

llvm/trunk/test/CodeGen/AArch64/arm64-shrink-wrapping.ll

 ; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: mov [[SUM:w[0-9]+]], wzr
 ; CHECK-NEXT: mov [[IV:w[0-9]+]], #10
 ;
 ; Next BB.
 ; CHECK: [[LOOP:LBB[0-9_]+]]: ; %for.body
 ; CHECK: bl _something
-; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: cbnz [[IV]], [[LOOP]]
 ;
 ; Next BB.
 ; Copy SUM into the returned register + << 3.
 ; CHECK: lsl w0, [[SUM]], #3
 ;
 ; Jump to epilogue.
 ; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]

 ; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x[0-9]+]], [sp, #-32]!
 ; CHECK-NEXT: stp [[CSR3:x[0-9]+]], [[CSR4:x[0-9]+]], [sp, #16]
 ; CHECK-NEXT: add [[NEW_SP:x[0-9]+]], sp, #16
 ; CHECK: mov [[SUM:w[0-9]+]], wzr
 ; CHECK-NEXT: mov [[IV:w[0-9]+]], #10
 ; Next BB.
 ; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
 ; CHECK: bl _something
-; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
 ; Next BB.
 ; CHECK: ; %for.end
 ; CHECK: mov w0, [[SUM]]
 ; CHECK-NEXT: ldp [[CSR3]], [[CSR4]], [sp, #16]
 ; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #32
 ; CHECK-NEXT: ret
 define i32 @freqSaveAndRestoreOutsideLoop2(i32 %cond) {

 ;
 ; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: mov [[SUM:w[0-9]+]], wzr
 ; CHECK-NEXT: mov [[IV:w[0-9]+]], #10
 ;
 ; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
 ; CHECK: bl _something
-; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
 ; Next BB.
 ; CHECK: bl _somethingElse
 ; CHECK-NEXT: lsl w0, [[SUM]], #3
 ;
 ; Jump to epilogue.
 ; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]
 ;

 ; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: bl _somethingElse
 ; CHECK-NEXT: mov [[SUM:w[0-9]+]], wzr
 ; CHECK-NEXT: mov [[IV:w[0-9]+]], #10
 ;
 ; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
 ; CHECK: bl _something
-; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: sub [[IV]], [[IV]], #1
+; CHECK-NEXT: add [[SUM]], w0, [[SUM]]
 ; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
 ; Next BB.
 ; CHECK: lsl w0, [[SUM]], #3
 ;
 ; Jump to epilogue.
 ; DISABLE: b [[EPILOG_BB:LBB[0-9_]+]]
 ;
 ; DISABLE: [[ELSE_LABEL]]: ; %if.else

 ; ENABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; CHECK: sub sp, sp, #16
 ; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Sum is merged with the returned register.
 ; CHECK: add [[VA_BASE:x[0-9]+]], sp, #16
-; CHECK-NEXT: str [[VA_BASE]], [sp, #8]
 ; CHECK-NEXT: cmp w1, #1
+; CHECK-NEXT: str [[VA_BASE]], [sp, #8]
+; CHECK-NEXT: mov [[SUM:w0]], wzr
 ; CHECK-NEXT: b.lt [[IFEND_LABEL:LBB[0-9_]+]]
-; CHECK: mov [[SUM:w0]], wzr
 ;
 ; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
 ; CHECK: ldr [[VA_ADDR:x[0-9]+]], [sp, #8]
 ; CHECK-NEXT: add [[NEXT_VA_ADDR:x[0-9]+]], [[VA_ADDR]], #8
 ; CHECK-NEXT: str [[NEXT_VA_ADDR]], [sp, #8]
 ; CHECK-NEXT: ldr [[VA_VAL:w[0-9]+]], {{\[}}[[VA_ADDR]]]
-; CHECK-NEXT: add [[SUM]], [[SUM]], [[VA_VAL]]
 ; CHECK-NEXT: sub w1, w1, #1
+; CHECK-NEXT: add [[SUM]], [[SUM]], [[VA_VAL]]
 ; CHECK-NEXT: cbnz w1, [[LOOP_LABEL]]
+; DISABLE-NEXT: b [[IFEND_LABEL]]
 ;
-; DISABLE-NEXT: b
 ; DISABLE: [[ELSE_LABEL]]: ; %if.else
 ; DISABLE: lsl w0, w1, #1
 ;
-; ENABLE: [[ELSE_LABEL]]: ; %if.else
-; ENABLE: lsl w0, w1, #1
-; ENABLE-NEXT: ret
-;
 ; CHECK: [[IFEND_LABEL]]:
 ; Epilogue code.
 ; CHECK: add sp, sp, #16
 ; CHECK-NEXT: ret
+;
+; ENABLE: [[ELSE_LABEL]]: ; %if.else
+; ENABLE-NEXT: lsl w0, w1, #1
+; ENABLE_NEXT: ret
 define i32 @variadicFunc(i32 %cond, i32 %count, ...) #0 {
 entry:
   %ap = alloca i8*, align 8
   %tobool = icmp eq i32 %cond, 0
   br i1 %tobool, label %if.else, label %if.then
 
 if.then:                                          ; preds = %entry
   %ap1 = bitcast i8** %ap to i8*

 ; CHECK: stp [[CSR1:x[0-9]+]], [[CSR2:x19]], [sp, #-16]!
 ;
 ; DISABLE: cbz w0, [[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: mov [[IV:w[0-9]+]], #10
 ;
 ; CHECK: [[LOOP_LABEL:LBB[0-9_]+]]: ; %for.body
 ; Inline asm statement.
-; CHECK: add x19, x19, #1
 ; CHECK: sub [[IV]], [[IV]], #1
-; CHECK-NEXT: cbnz [[IV]], [[LOOP_LABEL]]
+; CHECK: add x19, x19, #1
+; CHECK: cbnz [[IV]], [[LOOP_LABEL]]
 ; Next BB.
 ; CHECK: mov w0, wzr
 ; Epilogue code.
 ; CHECK-NEXT: ldp [[CSR1]], [[CSR2]], [sp], #16
 ; CHECK-NEXT: ret
 ; Next BB.
 ; CHECK: [[ELSE_LABEL]]: ; %if.else
 ; CHECK-NEXT: lsl w0, w1, #1

llvm/trunk/test/CodeGen/AArch64/misched-fusion.ll

-; RUN: llc -o - %s -mattr=+arith-cbz-fusion,+use-postra-scheduler | FileCheck %s
+; RUN: llc -o - %s -mattr=+arith-cbz-fusion | FileCheck %s
 ; RUN: llc -o - %s -mcpu=cyclone | FileCheck %s
 
 target triple = "arm64-apple-ios"
 
 declare void @foobar(i32 %v0, i32 %v1)
 
 ; Make sure sub is scheduled in front of cbnz
 ; CHECK-LABEL: test_sub_cbz:

llvm/trunk/test/CodeGen/AArch64/neg-imm.ll

; CHECK-LABEL:  %if.then3
   br i1 %cmp2, label %for.inc, label %if.then3
 
 if.then3:
   tail call void @foo(i32 %x.015)
   br label %for.inc
 
 for.inc:
 ; CHECK_LABEL: %for.inc
-; CHECK:  add
-; CHECK-NEXT:  cmp
-; CHECK:  b.le
+; CHECK:  cmp
+; CHECK-NEXT:  add
+; CHECK-NEXT:  b.le
 ; CHECK_LABEL: %for.cond.cleanup
   %inc = add nsw i32 %x.015, 1
   %cmp1 = icmp sgt i32 %x.015, %px
   br i1 %cmp1, label %for.cond.cleanup.loopexit, label %for.body
 
 for.cond.cleanup.loopexit:
   br label %for.cond.cleanup
 
 for.cond.cleanup:
   ret void
 }

llvm/trunk/test/CodeGen/AMDGPU/branch-relaxation.ll

 ; FIXME: Should be able to use s_cbranch_scc0
 ; GCN-LABEL: {{^}}long_backward_sbranch:
 ; GCN: v_mov_b32_e32 [[LOOPIDX:v[0-9]+]], 0{{$}}
 
 ; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: ; %bb2
 ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT: v_add_i32_e32 [[INC:v[0-9]+]], vcc, 1, [[LOOPIDX]]
 ; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 10, [[INC]]
+; GCN-NEXT: s_and_b64 vcc, exec, vcc
 
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: ;;#ASMEND
 
-; GCN-NEXT: s_and_b64 vcc, exec, vcc
 ; GCN-NEXT: s_cbranch_vccz [[ENDBB:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT: [[LONG_JUMP:BB[0-9]+_[0-9]+]]: ; %bb2
 ; GCN-NEXT: ; in Loop: Header=[[LOOPBB]] Depth=1
 ; GCN-NEXT: s_getpc_b64 vcc
 ; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONG_JUMP]]+4)-[[LOOPBB]]
 ; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0
 ; GCN-NEXT: s_setpc_b64 vcc

 
 ; GCN-NEXT: [[BRANCH_SKIP]]: ; %entry
 ; GCN-NEXT: s_getpc_b64 vcc
 ; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[RET]]-([[BRANCH_SKIP]]+4)
 ; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0
 ; GCN-NEXT: s_setpc_b64 vcc
 
 ; GCN-NEXT: [[LOOP_BODY]]: ; %loop_body
+; GCN: s_mov_b64 vcc, -1{{$}}
+; GCN: ;;#ASMSTART
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: ;;#ASMEND
-; GCN-NEXT: s_mov_b64 vcc, -1{{$}}
 ; GCN-NEXT: s_cbranch_vccz [[RET]]
 
 ; GCN-NEXT: [[LONGBB:BB[0-9]+_[0-9]+]]: ; %loop_body
 ; GCN-NEXT: ; in Loop: Header=[[LOOP_BODY]] Depth=1
 ; GCN-NEXT: s_getpc_b64 vcc
 ; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONGBB]]+4)-[[LOOP_BODY]]
 ; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0
 ; GCN-NEXT: s_setpc_b64 vcc

 ; GCN-DAG: v_cmp_gt_i32
 ; GCN: s_cbranch_vccnz
 
 ; GCN: s_setpc_b64
 ; GCN: s_setpc_b64
 
 ; GCN: [[LONG_BR_DEST0]]
 ; GCN: s_cmp_eq_u32
+; GCN-NEXT: ; implicit-def
 ; GCN-NEXT: s_cbranch_scc0
 ; GCN: s_setpc_b64
 
 ; GCN: s_endpgm
 define amdgpu_kernel void @long_branch_hang(i32 addrspace(1)* nocapture %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i64 %arg5) #0 {
 bb:
   %tmp = icmp slt i32 %arg2, 9
   %tmp6 = icmp eq i32 %arg1, 0

llvm/trunk/test/CodeGen/AMDGPU/cndmask-no-def-vcc.ll

 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare i1 @llvm.amdgcn.class.f32(float, i32)
 
 ; Produces error after adding an implicit def to v_cndmask_b32
 
 ; GCN-LABEL: {{^}}vcc_shrink_vcc_def:
 ; GCN: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}}
 ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc
-; GCN: v_cndmask_b32_e64 v1, 0, 1, s{{\[[0-9]+:[0-9]+\]}}
+; GCN: v_cndmask_b32_e64 v0, 0, 1, s{{\[[0-9]+:[0-9]+\]}}
 define void @vcc_shrink_vcc_def(float %arg, i32 %arg1, float %arg2, i32 %arg3) {
 bb0:
   %tmp = icmp sgt i32 %arg1, 4
   %c = icmp eq i32 %arg3, 0
   %tmp4 = select i1 %c, float %arg, float 1.000000e+00
   %tmp5 = fcmp ogt float %arg2, 0.000000e+00
   %tmp6 = fcmp olt float %arg2, 1.000000e+00
   %tmp7 = fcmp olt float %arg, %tmp4

 }
 
 ; The undef flag on the condition src must be preserved on the
 ; implicit vcc use to avoid verifier errors.
 
 ; GCN-LABEL: {{^}}preserve_condition_undef_flag:
 ; GCN-NOT: vcc
 ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc
-; GCN: v_cndmask_b32_e64 v1, 0, 1, s{{\[[0-9]+:[0-9]+\]}}
+; GCN: v_cndmask_b32_e64 v0, 0, 1, s{{\[[0-9]+:[0-9]+\]}}
 define void @preserve_condition_undef_flag(float %arg, i32 %arg1, float %arg2) {
 bb0:
   %tmp = icmp sgt i32 %arg1, 4
   %undef = call i1 @llvm.amdgcn.class.f32(float undef, i32 undef)
   %tmp4 = select i1 %undef, float %arg, float 1.000000e+00
   %tmp5 = fcmp ogt float %arg2, 0.000000e+00
   %tmp6 = fcmp olt float %arg2, 1.000000e+00
   %tmp7 = fcmp olt float %arg, %tmp4

llvm/trunk/test/CodeGen/AMDGPU/indirect-addressing-si.ll

 }
 
 ; GCN-LABEL: {{^}}extract_neg_offset_sgpr:
 ; The offset depends on the register that holds the first element of the vector.
 ; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0
 
 ; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
+; IDXMODE: v_mov_b32_e32 v2, 2
+; IDXMODE: v_mov_b32_e32 v3, 3
 ; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}}
 ; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE-NEXT: s_set_gpr_idx_off
 define void @extract_neg_offset_sgpr(i32 addrspace(1)* %out, i32 %offset) {
 entry:
   %index = add i32 %offset, -512
   %value = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
 
 ; GCN-LABEL: {{^}}extract_neg_offset_sgpr_loaded:
 ; The offset depends on the register that holds the first element of the vector.
 ; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0
 
 ; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
+; IDXMODE: v_mov_b32_e32 v0,
+; IDXMODE: v_mov_b32_e32 v1,
+; IDXMODE: v_mov_b32_e32 v2,
+; IDXMODE: v_mov_b32_e32 v3,
 ; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}}
 ; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE-NEXT: s_set_gpr_idx_off
 define void @extract_neg_offset_sgpr_loaded(i32 addrspace(1)* %out, <4 x i32> %vec0, <4 x i32> %vec1, i32 %offset) {
 entry:
   %index = add i32 %offset, -512
   %or = or <4 x i32> %vec0, %vec1
   %value = extractelement <4 x i32> %or, i32 %index

 
 ; GCN-DAG: v_mov_b32_e32 v[[VEC0_ELT0:[0-9]+]], 0x41880000
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41900000
 ; GCN-DAG: v_mov_b32_e32 v[[VEC0_ELT2:[0-9]+]], 0x41980000
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a00000
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a80000
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41b00000
 ; GCN-DAG: s_load_dword [[ARG:s[0-9]+]]
+; IDXMODE-DAG: s_add_i32 [[ARG_ADD:s[0-9]+]], [[ARG]], -16
 
 ; MOVREL-DAG: s_add_i32 m0, [[ARG]], -16
 ; MOVREL: v_movreld_b32_e32 v[[VEC0_ELT0]], 4.0
 ; GCN-NOT: m0
 
-; IDXMODE-DAG: s_add_i32 [[ARG_ADD:s[0-9]+]], [[ARG]], -16
 ; IDXMODE: s_set_gpr_idx_on [[ARG_ADD]], dst
 ; IDXMODE: v_mov_b32_e32 v[[VEC0_ELT0]], 4.0
 ; IDXMODE: s_set_gpr_idx_off
 
 ; GCN: v_mov_b32_e32 v[[VEC0_ELT2]], 0x4188cccd
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x4190cccd
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x4198cccd
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a0cccd

llvm/trunk/test/CodeGen/AMDGPU/skip-if-dead.ll

bb:
   br label %exit
 
 exit:
   ret void
 }
 
 ; CHECK-LABEL: {{^}}test_kill_control_flow_remainder:
 ; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0
+; CHECK-NEXT: v_mov_b32_e32 v{{[0-9]+}}, 0
 ; CHECK-NEXT: s_cbranch_scc1 [[RETURN_BB:BB[0-9]+_[0-9]+]]
 
 ; CHECK-NEXT: ; BB#1: ; %bb
 ; CHECK: v_mov_b32_e64 v7, -1
 ; CHECK: v_nop_e64
 ; CHECK: v_nop_e64
 ; CHECK: v_nop_e64
 ; CHECK: v_nop_e64

llvm/trunk/test/CodeGen/AMDGPU/wqm.ll

 ;
 ;CHECK-LABEL: {{^}}test_control_flow_3:
 ;CHECK-NEXT: ; %main_body
 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
 ;CHECK-NEXT: s_wqm_b64 exec, exec
 ;CHECK: image_sample
 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
 ;CHECK: image_sample
-;CHECK: store
 ;CHECK: v_cmp
+;CHECK: store
 define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, i32 %coord) {
 main_body:
   %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
   %tex.1 = bitcast <4 x float> %tex to <4 x i32>
   %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0)
   %dtex.1 = extractelement <4 x float> %dtex, i32 0
 
   call void @llvm.amdgcn.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)

llvm/trunk/test/CodeGen/ARM/arm-shrink-wrapping.ll

 ; CHECK-LABEL: debug_info:
 ;
 ; ENABLE: tst{{(\.w)?}}  r2, #1
 ; ENABLE-NEXT: beq      [[BB13:LBB[0-9_]+]]
 ;
 ; CHECK: push
 ;
 ; DISABLE: tst{{(\.w)?}}  r2, #1
+; DISABLE-NEXT: vst1.64
 ; DISABLE-NEXT: beq      [[BB13:LBB[0-9_]+]]
 ;
 ; CHECK: bl{{x?}} _pow
 ;
 ;
 ; ENABLE: pop
 ;
 ; CHECK: [[BB13]]:

llvm/trunk/test/CodeGen/PowerPC/ppc-shrink-wrapping.ll

 ; ENABLE: cmplwi 0, 3, 0
 ; ENABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; Make sure we save the link register 
 ; CHECK: mflr {{[0-9]+}}
 ;
 ; DISABLE: cmplwi 0, 3, 0
+; DISABLE-NEXT: std
+; DISABLE-NEXT: std
 ; DISABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Loop preheader
 ; CHECK-DAG: li [[SUM:[0-9]+]], 0
 ; CHECK-DAG: li [[IV:[0-9]+]], 10
 ; 
 ; Loop body
 ; CHECK: .[[LOOP:LBB[0-9_]+]]: # %for.body

 ; ENABLE: cmplwi 0, 3, 0
 ; ENABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; Make sure we save the link register
 ; CHECK: mflr {{[0-9]+}}
 ;
 ; DISABLE: cmplwi 0, 3, 0
+; DISABLE-NEXT: std
+; DISABLE-NEXT: std
 ; DISABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; CHECK: bl somethingElse
 ;
 ; Loop preheader
 ; CHECK-DAG: li [[SUM:[0-9]+]], 0
 ; CHECK-DAG: li [[IV:[0-9]+]], 10
 ; 

 ; ENABLE: cmplwi 0, 3, 0
 ; ENABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ;
 ; Prologue code.
 ; Make sure we save the CSR used in the inline asm: r14
 ; ENABLE-DAG: li [[IV:[0-9]+]], 10
 ; ENABLE-DAG: std 14, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
 ;
-; DISABLE: std 14, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
 ; DISABLE: cmplwi 0, 3, 0
+; DISABLE-NEXT: std 14, -[[STACK_OFFSET:[0-9]+]](1) # 8-byte Folded Spill
 ; DISABLE-NEXT: beq 0, .[[ELSE_LABEL:LBB[0-9_]+]]
 ; DISABLE: li [[IV:[0-9]+]], 10
 ;
 ; CHECK: nop
 ; CHECK: mtctr [[IV]]
 ;
 ; CHECK: .[[LOOP_LABEL:LBB[0-9_]+]]: # %for.body
 ; Inline asm statement.

llvm/trunk/test/CodeGen/SPARC/2011-01-19-DelaySlot.ll

 ;CHECK:      retl
 ;CHECK-NOT: restore
   ret i32 %a_addr.1.lcssa
 }
 
 define i32 @test_inlineasm(i32 %a) nounwind {
 entry:
 ;CHECK-LABEL:      test_inlineasm:
+;CHECK: cmp
 ;CHECK:      sethi
 ;CHECK:      !NO_APP
-;CHECK-NEXT: cmp
 ;CHECK-NEXT: ble
 ;CHECK-NEXT: mov
   tail call void asm sideeffect "sethi 0, %g0", ""() nounwind
   %0 = icmp slt i32 %a, 0
   br i1 %0, label %bb, label %bb1
 
 bb:                                               ; preds = %entry
   %1 = tail call i32 (...) @foo(i32 %a) nounwind

llvm/trunk/test/CodeGen/SystemZ/int-cmp-48.ll

exit:
   ret void
 }
 
 
 ; Check that we do not fold across an aliasing store.
 define void @f2(i8 *%src) {
 ; CHECK-LABEL: f2:
 ; CHECK: llc [[REG:%r[0-5]]], 0(%r2)
-; CHECK: mvi 0(%r2), 0
 ; CHECK: tmll [[REG]], 1
+; CHECK: mvi 0(%r2), 0
 ; CHECK: ber %r14
 ; CHECK: br %r14
 entry:
   %byte = load i8 , i8 *%src
   store i8 0, i8 *%src
   %and = and i8 %byte, 1
   %cmp = icmp eq i8 %and, 0
   br i1 %cmp, label %exit, label %store