[LLVM][Uniformity] Improve detection of uniform registers

[sameerds]

Summary

The MachineUA now queries the target to determine if a given register holds a
uniform value. This is determined using the corresponding register bank if
available, or by a combination of the register class and value type. This
assumes that the target is optimizing for performance by choosing registers, and
the target is responsible for any mismatch with the inferred uniformity.

For example, on AMDGPU, an SGPR is now treated as uniform, except if the
register bank is VCC (i.e., the register holds a wave-wide vector of 1-bit
values) or equivalently if it has a value type of s1.

• This does not always work with inline asm, where the register bank or the value type might not be present. We assume that the SGPR is uniform, because it is not expected to be s1 in the vast majority of cases.
• The pseudo branch instruction SI_LOOP is now hard-coded to be always divergent, although its condition is an SGPR.

[arsenm]

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

2879

Can combine these two queries into getRegClassOrRegBank

2891

Just use LLT instead of auto, it's even shorter

2892–2893

You should just be able to do equality without pre-checking for a valid type. Just MRI.getType() != LLT::scalar(1) should be enough

2895

Don't see why this needs to be an assert, equality with s1 should be adequate

[sameerds]

• Simplified comparison of value type with s1.
• Ported the test for temporal divergence from the LLVM version to MIR.

[sameerds]

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

2879

What's the advantage of that? The PointerUnion will simply conflate two different things for me, and the only improvement that I see is that I can check both for nullptr at the same time. After that, I still have to insert at least one ugly dyn_cast along with a full type name instead of auto just to get to the RegBank. Isn't the current version much more readable?

[arsenm]

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

2879

It's one map lookup instead of 2. It also is really one thing, not two

[sameerds]

simplified inspection of registers; let RBI handle that

[sameerds]

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

2879

Managed to just offload the decision to RegisterBankInfo::getRegBank(). It already handles all the corner cases, and now we treat it as the source of truth rather than duplicating that logic.

[ruiling]

llvm/lib/Target/AMDGPU/SIInstructions.td

423

I think `SI_IF and SI_ELSE also need this flag being set, but it can be done in a separate change with proper tests.

[Petar.Avramovic]

llvm/lib/Target/AMDGPU/SIInstructions.td

423

I think the same. Marking SI_IF and SI_ELSE as never uniform fixed test failures I was seeing.

llvm/include/llvm/ADT/GenericUniformityImpl.h

public:
 
   /// \brief Mark \p UniVal as a value that is always uniform.
   void addUniformOverride(const InstructionT &Instr);
 
   /// \brief Mark \p DivVal as a value that is always divergent.
   /// \returns Whether the tracked divergence state of \p DivVal changed.
   bool markDivergent(const InstructionT &I);
   bool markDivergent(ConstValueRefT DivVal);
-  bool markDefsDivergent(const InstructionT &Instr,
-                         bool AllDefsDivergent = true);
+  bool markDefsDivergent(const InstructionT &Instr);
 
   /// \brief Propagate divergence to all instructions in the region.
   /// Divergence is seeded by calls to \p markDivergent.
   void compute();
 
   /// \brief Whether any value was marked or analyzed to be divergent.
   bool hasDivergence() const { return !DivergentValues.empty(); }
 

llvm/include/llvm/CodeGen/RegisterBankInfo.h

public:
   ///
   /// \pre Reg != 0 (NoRegister)
   const RegisterBank *getRegBank(Register Reg, const MachineRegisterInfo &MRI,
                                  const TargetRegisterInfo &TRI) const;
 
   /// Get the total number of register banks.
   unsigned getNumRegBanks() const { return NumRegBanks; }
 
+  /// Returns true if the register bank is considered divergent.
+  virtual bool isDivergentRegBank(const RegisterBank *RB) const {
+    return false;
+  }
+
   /// Get a register bank that covers \p RC.
   ///
   /// \pre \p RC is a user-defined register class (as opposed as one
   /// generated by TableGen).
   ///
   /// \note The mapping RC -> RegBank could be built while adding the
   /// coverage for the register banks. However, we do not do it, because,
   /// at least for now, we only need this information for register classes

llvm/include/llvm/CodeGen/TargetRegisterInfo.h

 #ifndef LLVM_CODEGEN_TARGETREGISTERINFO_H
 #define LLVM_CODEGEN_TARGETREGISTERINFO_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineValueType.h"
+#include "llvm/CodeGen/RegisterBank.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/MC/LaneBitmask.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Printable.h"
 #include <cassert>
 #include <cstdint>
public:
   /// function.  Used by MachineRegisterInfo::isConstantPhysReg().
   virtual bool isConstantPhysReg(MCRegister PhysReg) const { return false; }
 
   /// Returns true if the register class is considered divergent.
   virtual bool isDivergentRegClass(const TargetRegisterClass *RC) const {
     return false;
   }
 
+  /// Returns true if the register is considered uniform.
+  virtual bool isUniformReg(const MachineRegisterInfo &MRI,
+                            const RegisterBankInfo &RBI, Register Reg) const {
+    return false;
+  }
+
   /// Physical registers that may be modified within a function but are
   /// guaranteed to be restored before any uses. This is useful for targets that
   /// have call sequences where a GOT register may be updated by the caller
   /// prior to a call and is guaranteed to be restored (also by the caller)
   /// after the call.
   virtual bool isCallerPreservedPhysReg(MCRegister PhysReg,
                                         const MachineFunction &MF) const {
     return false;

llvm/lib/Analysis/UniformityAnalysis.cpp

 template <>
 bool llvm::GenericUniformityAnalysisImpl<SSAContext>::hasDivergentDefs(
     const Instruction &I) const {
   return isDivergent((const Value *)&I);
 }
 
 template <>
 bool llvm::GenericUniformityAnalysisImpl<SSAContext>::markDefsDivergent(
-    const Instruction &Instr, bool AllDefsDivergent) {
+    const Instruction &Instr) {
   return markDivergent(&Instr);
 }
 
 template <> void llvm::GenericUniformityAnalysisImpl<SSAContext>::initialize() {
   for (auto &I : instructions(F)) {
     if (TTI->isSourceOfDivergence(&I))
       markDivergent(I);
     else if (TTI->isAlwaysUniform(&I))

llvm/lib/CodeGen/MachineUniformityAnalysis.cpp

for (auto &op : I.operands()) {
     if (isDivergent(op.getReg()))
       return true;
   }
   return false;
 }
 
 template <>
 bool llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::markDefsDivergent(
-    const MachineInstr &Instr, bool AllDefsDivergent) {
+    const MachineInstr &Instr) {
   bool insertedDivergent = false;
   const auto &MRI = F.getRegInfo();
+  const auto &RBI = *F.getSubtarget().getRegBankInfo();
   const auto &TRI = *MRI.getTargetRegisterInfo();
   for (auto &op : Instr.operands()) {
     if (!op.isReg() || !op.isDef())
       continue;
     if (!op.getReg().isVirtual())
       continue;
     assert(!op.getSubReg());
-    if (!AllDefsDivergent) {
-      auto *RC = MRI.getRegClassOrNull(op.getReg());
-      if (RC && !TRI.isDivergentRegClass(RC))
-        continue;
-    }
+    if (TRI.isUniformReg(MRI, RBI, op.getReg()))
     insertedDivergent |= markDivergent(op.getReg());
   }
   return insertedDivergent;
 }
 
 template <>
 void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::initialize() {
   const auto &InstrInfo = *F.getSubtarget().getInstrInfo();
 
   for (const MachineBasicBlock &block : F) {
     for (const MachineInstr &instr : block) {
       auto uniformity = InstrInfo.getInstructionUniformity(instr);
       if (uniformity == InstructionUniformity::AlwaysUniform) {
         addUniformOverride(instr);
         continue;
       }
 
       if (uniformity == InstructionUniformity::NeverUniform) {
-        markDefsDivergent(instr, /* AllDefsDivergent = */ false);
+        if (markDivergent(instr))
+          Worklist.push_back(&instr);
       }
     }
   }
 }
 
 template <>
 void llvm::GenericUniformityAnalysisImpl<MachineSSAContext>::pushUsers(
     Register Reg) {

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h

public:
 
   const InstructionMapping &getImageMapping(const MachineRegisterInfo &MRI,
                                             const MachineInstr &MI,
                                             int RsrcIdx) const;
 
 public:
   AMDGPURegisterBankInfo(const GCNSubtarget &STI);
 
+  bool isDivergentRegBank(const RegisterBank *RB) const override;
+
   unsigned copyCost(const RegisterBank &A, const RegisterBank &B,
                     unsigned Size) const override;
 
   unsigned getBreakDownCost(const ValueMapping &ValMapping,
                             const RegisterBank *CurBank = nullptr) const override;
 
   const RegisterBank &getRegBankFromRegClass(const TargetRegisterClass &RC,
                                              LLT) const override;

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
   llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);
 }
 
 static bool isVectorRegisterBank(const RegisterBank &Bank) {
   unsigned BankID = Bank.getID();
   return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
 }
 
+bool AMDGPURegisterBankInfo::isDivergentRegBank(const RegisterBank *RB) const {
+  return RB != &AMDGPU::SGPRRegBank;
+}
+
 unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
                                           const RegisterBank &Src,
                                           unsigned Size) const {
   // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
   if (Dst.getID() == AMDGPU::SGPRRegBankID &&
       (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
     return std::numeric_limits<unsigned>::max();
   }

llvm/lib/Target/AMDGPU/SIInstructions.td

 }
 
 def SI_LOOP : CFPseudoInstSI <
   (outs), (ins SReg_1:$saved, brtarget:$target),
   [(AMDGPUloop i1:$saved, bb:$target)], 1, 1> {
   let Size = 8;
   let isBranch = 1;
   let hasSideEffects = 1;
+  let IsNeverUniform = 1;
 }
 
 } // End isTerminator = 1
 
 def SI_END_CF : CFPseudoInstSI <
   (outs), (ins SReg_1:$saved), [], 1, 1> {
   let Size = 4;
   let isAsCheapAsAMove = 1;

llvm/lib/Target/AMDGPU/SIRegisterInfo.h

getRegClassForOperandReg(const MachineRegisterInfo &MRI,
                            const MachineOperand &MO) const;
 
   bool isVGPR(const MachineRegisterInfo &MRI, Register Reg) const;
   bool isAGPR(const MachineRegisterInfo &MRI, Register Reg) const;
   bool isVectorRegister(const MachineRegisterInfo &MRI, Register Reg) const {
     return isVGPR(MRI, Reg) || isAGPR(MRI, Reg);
   }
 
+  // FIXME: SGPRs are assumed to be uniform, but this is not true for i1 SGPRs
+  // (such as VCC) which hold a wave-wide vector of boolean values. Examining
+  // just the register class is not suffcient; it needs to be combined with a
+  // value type. The next predicate isUniformReg() does this correctly.
   bool isDivergentRegClass(const TargetRegisterClass *RC) const override {
     return !isSGPRClass(RC);
   }
 
+  bool isUniformReg(const MachineRegisterInfo &MRI, const RegisterBankInfo &RBI,
+                    Register Reg) const override;
+
   ArrayRef<int16_t> getRegSplitParts(const TargetRegisterClass *RC,
                                      unsigned EltSize) const;
 
   bool shouldCoalesce(MachineInstr *MI,
                       const TargetRegisterClass *SrcRC,
                       unsigned SubReg,
                       const TargetRegisterClass *DstRC,
                       unsigned DstSubReg,

llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp

MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
   } else {
     for (MCRegister Reg : *RC)
       if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
         return Reg;
   }
   return MCRegister();
 }
 
+bool SIRegisterInfo::isUniformReg(const MachineRegisterInfo &MRI,
+                                  const RegisterBankInfo &RBI,
+                                  Register Reg) const {
+  auto *RB = RBI.getRegBank(Reg, MRI, *MRI.getTargetRegisterInfo());
+  if (!RB)
+    return false;
+
+  return !RBI.isDivergentRegBank(RB);
+}
+
 ArrayRef<int16_t> SIRegisterInfo::getRegSplitParts(const TargetRegisterClass *RC,
                                                    unsigned EltSize) const {
   const unsigned RegBitWidth = AMDGPU::getRegBitWidth(*RC->MC);
   assert(RegBitWidth >= 32 && RegBitWidth <= 1024);
 
   const unsigned RegDWORDs = RegBitWidth / 32;
   const unsigned EltDWORDs = EltSize / 4;
   assert(RegSplitParts.size() + 1 >= EltDWORDs);
 
   const std::vector<int16_t> &Parts = RegSplitParts[EltDWORDs - 1];
   const unsigned NumParts = RegDWORDs / EltDWORDs;
 
   return ArrayRef(Parts.data(), NumParts);
 }
 
 const TargetRegisterClass*
 SIRegisterInfo::getRegClassForReg(const MachineRegisterInfo &MRI,
                                   Register Reg) const {
   return Reg.isVirtual() ? MRI.getRegClass(Reg) : getPhysRegBaseClass(Reg);
 }
 
 const TargetRegisterClass *
 SIRegisterInfo::getRegClassForOperandReg(const MachineRegisterInfo &MRI,
                                          const MachineOperand &MO) const {
   const TargetRegisterClass *SrcRC = getRegClassForReg(MRI, MO.getReg());
   return getSubRegisterClass(SrcRC, MO.getSubReg());

llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/always-uniform-gmir.mir

registers:
   - { id: 0, class: _, preferred-register: '' }
   - { id: 1, class: sreg_32, preferred-register: '' }
   - { id: 2, class: vgpr_32, preferred-register: '' }
   - { id: 3, class: _, preferred-register: '' }
 body:             |
   bb.0:
     liveins: $vgpr0
     ; CHECK-LABEL: MachineUniformityInfo for function: asm_sgpr
-    ; FIXME: This is backwards
-    ; CHECK: DIVERGENT: %1
+    ; CHECK-NOT: DIVERGENT: %1
 
     %0:_(s32) = COPY $vgpr0
     %2:vgpr_32 = COPY %0(s32)
     INLINEASM &"; def $0, $1", 0 /* attdialect */, 1966090 /* regdef:SReg_32 */, def %1, 1835017 /* reguse:VGPR_32 */, %2
     %3:_(s32) = COPY %1
     $vgpr0 = COPY %3(s32)
     SI_RETURN implicit $vgpr0
 

llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/atomics.mir

 # RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
 
 ---
 name:            test1
 tracksRegLiveness: true
 body:             |
   bb.0:
     ; CHECK-LABEL: MachineUniformityInfo for function: test1
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %3:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %5:vreg_64 = COPY %3
     %6:vreg_64 = COPY %3
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: FLAT_ATOMIC_SWAP_RTN
+    ; CHECK: DIVERGENT{{.*}}FLAT_ATOMIC_SWAP_RTN
     %4:vgpr_32 = FLAT_ATOMIC_SWAP_RTN killed %5, %2, 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst (s32))
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: FLAT_ATOMIC_SWAP_RTN
+    ; CHECK: DIVERGENT{{.*}}FLAT_ATOMIC_SWAP_RTN
     %7:vgpr_32 = FLAT_ATOMIC_SWAP_RTN killed %6, %2, 0, 1, implicit $exec, implicit $flat_scr ; No memopernads
     $vgpr0 = COPY %4
     SI_RETURN implicit $vgpr0
 ...
 
 ---
 name:            test2
 tracksRegLiveness: true
 body:             |
   bb.0:
     ; CHECK-LABEL: MachineUniformityInfo for function: test2
     %3:vgpr_32 = IMPLICIT_DEF
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %4:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %5:sreg_64 = REG_SEQUENCE %3, %subreg.sub0, %2, %subreg.sub1
     %7:vreg_64 = COPY %4
     %8:vreg_64 = COPY %5
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: FLAT_ATOMIC_CMPSWAP_RTN
+    ; CHECK: DIVERGENT{{.*}}FLAT_ATOMIC_CMPSWAP_RTN
     %6:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN killed %7, killed %8, 0, 1, implicit $exec, implicit $flat_scr :: (load store seq_cst seq_cst (s32))
     %9:sreg_64_xexec = V_CMP_EQ_U32_e64 %6, %2, implicit $exec
     %10:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
     $vgpr0 = COPY %6
     $vgpr1 = COPY %10
     SI_RETURN implicit $vgpr0, implicit $vgpr1
 ...
 
 ---
 name:            atomic_inc
 tracksRegLiveness: true
 body:             |
   bb.0:
   ; CHECK-LABEL: MachineUniformityInfo for function: atomic_inc
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %3:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %5:vreg_64 = COPY %3
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: GLOBAL_ATOMIC_INC_RTN
+    ; CHECK: DIVERGENT{{.*}}GLOBAL_ATOMIC_INC_RTN
     %4:vgpr_32 = GLOBAL_ATOMIC_INC_RTN killed %5, %2, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
     $vgpr0 = COPY %4
     SI_RETURN implicit $vgpr0
 ...
 
 ---
 name:            atomic_inc_64
 tracksRegLiveness: true
 body:             |
   bb.0:
   ; CHECK-LABEL: MachineUniformityInfo for function: atomic_inc_64
     %3:vgpr_32 = IMPLICIT_DEF
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %4:sreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
     %5:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %7:vreg_64 = COPY %5
     %8:vreg_64 = COPY %4
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: GLOBAL_ATOMIC_INC_X2_RTN
+    ; CHECK: DIVERGENT{{.*}}GLOBAL_ATOMIC_INC_X2_RTN
     %6:vreg_64 = GLOBAL_ATOMIC_INC_X2_RTN killed %7, killed %8, 0, 1, implicit $exec :: (load store (s64), addrspace 1)
     %9:vgpr_32 = COPY %6.sub1
     %10:vgpr_32 = COPY %6.sub0
     $vgpr0 = COPY %10
     $vgpr1 = COPY %9
     SI_RETURN implicit $vgpr0, implicit $vgpr1
 ...
 
 ---
 name:            atomic_dec
 tracksRegLiveness: true
 body:             |
   bb.0:
   ; CHECK-LABEL: MachineUniformityInfo for function: atomic_dec
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %3:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %5:vreg_64 = COPY %3
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: GLOBAL_ATOMIC_DEC_RTN
+    ; CHECK: DIVERGENT{{.*}}GLOBAL_ATOMIC_DEC_RTN
     %4:vgpr_32 = GLOBAL_ATOMIC_DEC_RTN killed %5, %2, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
     $vgpr0 = COPY %4
     SI_RETURN implicit $vgpr0
 ...
 
 
 ---
 name:            atomic_dec_64
 tracksRegLiveness: true
 body:             |
   bb.0:
   ; CHECK-LABEL: MachineUniformityInfo for function: atomic_dec_64
     %3:vgpr_32 = IMPLICIT_DEF
     %2:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %0:vgpr_32 = IMPLICIT_DEF
     %4:sreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
     %5:sreg_64 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1
     %7:vreg_64 = COPY %5
     %8:vreg_64 = COPY %4
-    ; CHECK: DIVERGENT
-    ; CHECK-SAME: GLOBAL_ATOMIC_DEC_X2_RTN
+    ; CHECK: DIVERGENT{{.*}}GLOBAL_ATOMIC_DEC_X2_RTN
     %6:vreg_64 = GLOBAL_ATOMIC_DEC_X2_RTN killed %7, killed %8, 0, 1, implicit $exec :: (load store (s64), addrspace 1)
     %9:vgpr_32 = COPY %6.sub1
     %10:vgpr_32 = COPY %6.sub0
     $vgpr0 = COPY %10
     $vgpr1 = COPY %9
     SI_RETURN implicit $vgpr0, implicit $vgpr1
 ...

llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/hidden-loop-diverge.mir

bb.0:
         %3:_(s1) = G_ICMP intpred(slt), %0(s32), %1
         G_BRCOND %3(s1), %bb.3                            ; Uniform branch
         G_BR %bb.1
     bb.1:
         successors: %bb.3, %bb.2
 
         %4:_(s32) = G_PHI %1(s32), %bb.0, %7(s32), %bb.2
         %5:_(s1) = G_ICMP intpred(slt), %1(s32), %2(s32)
-        G_BRCOND %5(s1), %bb.3
+        G_BRCOND %5(s1), %bb.3                            ; Divergent exit
         G_BR %bb.2
     bb.2:
         successors: %bb.4, %bb.1
 
         %6:_(s32) = G_CONSTANT i32 1
         %7:_(s32) = G_ADD %6(s32), %4(s32)
         %8:_(s1) = G_ICMP intpred(sgt), %2(s32), %1(s32)
-        G_BRCOND %8(s1), %bb.4
+        G_BRCOND %8(s1), %bb.4                            ; Divergent exit
         G_BR %bb.1
     bb.3:
         successors: %bb.4, %bb.5
 
         %9:_(s32) = G_PHI %20(s32), %bb.0, %4(s32), %bb.1    ; Temporal divergent phi
         G_BRCOND %3(s1), %bb.4
         G_BR %bb.5
 
     bb.4:
         successors: %bb.5
 
-        %10:_(s32) = G_PHI %21(s32), %bb.3, %22(s32), %bb.2
+        %10:_(s32) = G_PHI %21(s32), %bb.3, %22(s32), %bb.2  ; Temporal divergent phi
         G_BR %bb.5
     bb.5:
         %11:_(s32) = G_PHI %20(s32), %bb.3, %22(s32), %bb.4
 ...

llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/never-uniform.mir

 ---
 name:            writelane
 machineFunctionInfo:
   isEntryFunction: true
 body:             |
   bb.0:
     ; CHECK-LABEL: MachineUniformityInfo for function: writelane
     ; CHECK: DIVERGENT: %4
-    ; CHECK: DIVERGENT: %5
+
+    ; Note how %5 is the result of a vector compare, but it is reported as
+    ; uniform because it is stored in an sreg.
+    ; CHECK-NOT: DIVERGENT: %5
+
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
     %2:sgpr_32 = V_READFIRSTLANE_B32 %0, implicit $exec
     %3:sgpr_32 = V_READLANE_B32 %1, 0, implicit $exec
     $sgpr0 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
     $sgpr1 = V_READLANE_B32 $vgpr1, $sgpr0, implicit $exec
 
     %4:vgpr_32 = V_WRITELANE_B32 0, 0, %0, implicit $exec

llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/temporal-divergence.mir

+# RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
+
+---
+# CHECK-LABEL: MachineUniformityInfo for function: temporal_diverge
+name:            temporal_diverge
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    %15:_(s64) = G_CONSTANT i64 0
+
+  bb.2:
+    successors: %bb.3, %bb.2
+
+    %11:_(s64) = G_PHI %12(s64), %bb.2, %15(s64), %bb.1
+    %18:_(s1) = G_CONSTANT i1 false
+    %12:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %18(s1), %11(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %12(s64), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %14:_(s64) = G_PHI %12(s64), %bb.2
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s64)
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: phi_at_exit
+name:            phi_at_exit
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    successors: %bb.2, %bb.3
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %7:_(p4) = COPY %3(p4)
+    %8:_(s64) = G_CONSTANT i64 40
+    %9:_(p4) = G_PTR_ADD %7, %8(s64)
+    %10:_(s32) = G_LOAD %9(p4) :: (load (s32), addrspace 4)
+    %11:_(s32) = G_CONSTANT i32 0
+    %12:_(s1) = G_ICMP intpred(sge), %10(s32), %11
+    G_BRCOND %12(s1), %bb.3
+    G_BR %bb.2
+
+  bb.2:
+    %24:_(s64) = G_CONSTANT i64 0
+    %14:_(s1) = G_CONSTANT i1 false
+    G_BR %bb.4
+
+  bb.3:
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5, %bb.4
+
+    %15:_(s64) = G_PHI %24(s64), %bb.2, %16(s64), %bb.4
+    %16:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %14(s1), %15(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %16(s64), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.5:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %18:_(s64) = G_PHI %16(s64), %bb.4
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %18(s64)
+    G_BR %bb.3
+
+  bb.6:
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: phi_after_exit
+name:            phi_after_exit
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    successors: %bb.2, %bb.3
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %7:_(p4) = COPY %3(p4)
+    %8:_(s64) = G_CONSTANT i64 40
+    %9:_(p4) = G_PTR_ADD %7, %8(s64)
+    %10:_(s32) = G_LOAD %9(p4) :: (dereferenceable invariant load (s32), addrspace 4)
+    %11:_(s32) = G_CONSTANT i32 0
+    %12:_(s1) = G_ICMP intpred(sge), %10(s32), %11
+    G_BRCOND %12(s1), %bb.3
+    G_BR %bb.2
+
+  bb.2:
+    %24:_(s64) = G_CONSTANT i64 0
+    %14:_(s1) = G_CONSTANT i1 false
+    G_BR %bb.4
+
+  bb.3:
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5, %bb.4
+
+    %15:_(s64) = G_PHI %24(s64), %bb.2, %16(s64), %bb.4
+    %16:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %14(s1), %15(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %16(s64), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.5:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %18:_(s64) = G_PHI %16(s64), %bb.4
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %18(s64)
+    G_BR %bb.3
+
+  bb.6:
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: temporal_diverge_inloop
+name:            temporal_diverge_inloop
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %7:_(p4) = COPY %3(p4)
+    %8:_(s64) = G_CONSTANT i64 40
+    %9:_(p4) = G_PTR_ADD %7, %8(s64)
+    %10:_(s32) = G_LOAD %9(p4) :: (dereferenceable invariant load (s32), addrspace 4)
+    %12:_(s32) = G_CONSTANT i32 0
+    %13:_(s1) = G_ICMP intpred(slt), %10(s32), %12
+
+  bb.2:
+    %25:_(s64) = G_CONSTANT i64 0
+
+  bb.3:
+    successors: %bb.4, %bb.3
+
+    %15:_(s64) = G_PHI %25(s64), %bb.2, %16(s64), %bb.3
+    %24:_(s1) = G_CONSTANT i1 false
+    %16:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %24(s1), %15(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %16(s64), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    successors: %bb.5, %bb.2
+
+    %18:_(s64) = G_PHI %16(s64), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %18(s64)
+    G_BRCOND %13(s1), %bb.2
+    G_BR %bb.5
+
+  bb.5:
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: temporal_uniform_indivloop
+name:            temporal_uniform_indivloop
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %19:_(s64) = G_CONSTANT i64 0
+    %7:_(p4) = COPY %3(p4)
+    %8:_(s64) = G_CONSTANT i64 40
+    %9:_(p4) = G_PTR_ADD %7, %8(s64)
+    %10:_(s32) = G_LOAD %9(p4) :: (dereferenceable invariant load (s32), addrspace 4)
+    %12:_(s32) = G_CONSTANT i32 0
+    %13:_(s1) = G_ICMP intpred(sge), %10(s32), %12
+
+  bb.2:
+    %15:_(s64) = G_PHI %16(s64), %bb.4, %19(s64), %bb.1
+    %24:_(s1) = G_CONSTANT i1 true
+    %16:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %24(s1), %15(s64)
+
+  bb.3:
+    successors: %bb.4, %bb.3
+
+    G_BRCOND %13(s1), %bb.3
+    G_BR %bb.4
+
+  bb.4:
+    successors: %bb.5, %bb.2
+
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %16(s64), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.5:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %18:_(s64) = G_PHI %16(s64), %bb.4
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %18(s64)
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: temporal_diverge_loopuser
+name:            temporal_diverge_loopuser
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %19:_(s64) = G_CONSTANT i64 0
+
+  bb.2:
+    successors: %bb.3, %bb.2
+
+    %10:_(s64) = G_PHI %11(s64), %bb.2, %19(s64), %bb.1
+    %24:_(s1) = G_CONSTANT i1 false
+    %11:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %24(s1), %10(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %11(s64), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+  ; CHECK-NOT:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %13:_(s64) = G_PHI %11(s64), %bb.2
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %13(s64)
+    %14:_(p4) = COPY %3(p4)
+    %15:_(s64) = G_CONSTANT i64 40
+    %16:_(p4) = G_PTR_ADD %14, %15(s64)
+    %17:_(s32) = G_LOAD %16(p4) :: (dereferenceable invariant load (s32), addrspace 4)
+    %25:_(s32) = G_CONSTANT i32 0
+    %18:_(s1) = G_ICMP intpred(slt), %17(s32), %25
+
+  bb.4:
+    successors: %bb.5, %bb.4
+
+    G_BRCOND %18(s1), %bb.4
+    G_BR %bb.5
+
+  bb.5:
+    S_ENDPGM 0
+
+...
+---
+# CHECK-LABEL: MachineUniformityInfo for function: temporal_diverge_loopuser_nested
+name:            temporal_diverge_loopuser_nested
+alignment:       1
+legalized:       true
+tracksRegLiveness: true
+registers:
+  - { id: 3, class: _ }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: sgpr_32 }
+  - { id: 6, class: sgpr_32 }
+liveins:
+  - { reg: '$sgpr0_sgpr1', virtual-reg: '%3' }
+  - { reg: '$vgpr0', virtual-reg: '%4' }
+  - { reg: '$sgpr2', virtual-reg: '%5' }
+  - { reg: '$sgpr3', virtual-reg: '%6' }
+body:             |
+  bb.1:
+    liveins: $sgpr0_sgpr1
+
+    %3:_(p4) = COPY $sgpr0_sgpr1
+    %7:_(p4) = COPY %3(p4)
+    %8:_(s64) = G_CONSTANT i64 40
+    %9:_(p4) = G_PTR_ADD %7, %8(s64)
+    %10:_(s32) = G_LOAD %9(p4) :: (dereferenceable invariant load (s32), addrspace 4)
+    %12:_(s32) = G_CONSTANT i32 0
+    %13:_(s1) = G_ICMP intpred(sge), %10(s32), %12
+
+  bb.2:
+    %23:_(s64) = G_CONSTANT i64 0
+
+  bb.3:
+    successors: %bb.4, %bb.3
+
+    %15:_(s64) = G_PHI %23(s64), %bb.2, %16(s64), %bb.3
+    %25:_(s1) = G_CONSTANT i1 false
+    %16:sreg_64_xexec(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.if.break), %25(s1), %15(s64)
+  ; CHECK:   DIVERGENT: SI_LOOP
+    SI_LOOP %16(s64), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+  ; CHECK:   DIVERGENT: %{{[0-9]+}}: %{{[0-9]+}}:_(s64) = G_PHI
+    %18:_(s64) = G_PHI %16(s64), %bb.3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %18(s64)
+
+  bb.5:
+
+  bb.6:
+    successors: %bb.8, %bb.5
+
+    G_BRCOND %13(s1), %bb.8
+    G_BR %bb.5
+
+  bb.7:
+    S_ENDPGM 0
+
+  bb.8:
+    successors: %bb.7, %bb.2
+
+    %24:_(s1) = G_CONSTANT i1 false
+    G_BRCOND %24(s1), %bb.7
+    G_BR %bb.2
+
+...

llvm/test/Analysis/UniformityAnalysis/AMDGPU/deprecated/hidden-diverge.mir

 # RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
 
+# This test was generated using SelectionDAG, where the compilation flow does
+# not match the assumptions made in MachineUA. For now, this test mostly serves
+# the purpose of catching in any crash when invoking MachineUA. The test should
+# be deleted when it is clear that it is not actually testing anything useful.
+
 ---
 # CHECK-LABEL: MachineUniformityInfo for function: hidden_diverge
 # CHECK-LABEL: BLOCK bb.0
 # CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:vgpr_32(s32) = COPY $vgpr0
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:sreg_64 = V_CMP_GT_I32_e64
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:sreg_64 = V_CMP_LT_I32_e64
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:vreg_1 = COPY
-# CHECK: DIVERGENT: %{{[0-9]*}}:sreg_64 = SI_IF
-# CHECK: DIVERGENT: S_BRANCH %bb.1
 # CHECK-LABEL: BLOCK bb.2
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:sreg_32 = PHI %{{[0-9]*}}:sreg_32, %bb.0, %{{[0-9]*}}:sreg_32, %bb.1
 # CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:vreg_1 = PHI %{{[0-9]*}}:vreg_1, %bb.0, %{{[0-9]*}}:sreg_64, %bb.1
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:sreg_64 = COPY %{{[0-9]*}}:vreg_1
-# CHECK: DIVERGENT: %{{[0-9]*}}:sreg_64 = SI_IF %{{[0-9]*}}:sreg_64, %bb.4
-# CHECK: DIVERGENT: S_BRANCH %bb.3
 # CHECK-LABEL: BLOCK bb.3
 # CHECK-LABEL: BLOCK bb.4
-# CHECK: DIVERGENT: %{{[0-9]*}}: %{{[0-9]*}}:vgpr_32 = PHI %{{[0-9]*}}:sreg_32, %bb.2, %{{[0-9]*}}:sreg_32, %bb.3
 
 name:            hidden_diverge
 tracksRegLiveness: true
 body:             |
   bb.0:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
     liveins: $vgpr0, $sgpr0_sgpr1
 

llvm/test/Analysis/UniformityAnalysis/AMDGPU/deprecated/irreducible-1.mir

 # RUN: llc -mtriple=amdgcn-- -run-pass=print-machine-uniformity -o - %s 2>&1 | FileCheck %s
 
+# This test was generated using SelectionDAG, where the compilation flow does
+# not match the assumptions made in MachineUA. For now, this test mostly serves
+# the purpose of catching in any crash when invoking MachineUA. The test should
+# be deleted when it is clear that it is not actually testing anything useful.
+
 # CHECK-LABEL: MachineUniformityInfo for function: irreducible
-# CHECK: CYCLES ASSSUMED DIVERGENT:
-# CHECK:   depth=1: entries(bb.2 bb.1) bb.3 bb.5 bb.4
-# CHECK: CYCLES WITH DIVERGENT EXIT:
-# CHECK-DAG:   depth=1: entries(bb.2 bb.1) bb.3 bb.5 bb.4
-# CHECK-DAG:   depth=2: entries(bb.3 bb.1) bb.5 bb.4
 
 ---
 name:            irreducible
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
 body:             |
   bb.0:

llvm/test/Analysis/UniformityAnalysis/AMDGPU/temporal_diverge.ll

H:
   %uni.merge.h = phi i32 [ 0, %G ], [ %uni.inc, %H ]
   %uni.inc = add i32 %uni.merge.h, 1
   %div.exitx = icmp slt i32 %tid, 0
   br i1 %div.exitx, label %X, label %H ; divergent branch
 ; CHECK: DIVERGENT: %div.exitx =
 ; CHECK: DIVERGENT: br i1 %div.exitx,
 
 X:
+; CHECK: DIVERGENT: %div.user =
   %div.user = add i32 %uni.inc, 5
   br i1 %uni.cond, label %G, label %Y
 
 Y:
+; CHECK: DIVERGENT: %div.alsouser =
   %div.alsouser = add i32 %uni.inc, 5
   ret void
 }
 
 
 ; temporal-uniform use of a value, definition and users are carried by a
 ; surrounding divergent loop
 define amdgpu_kernel void @temporal_uniform_indivloop(i32 %n, i32 %a, i32 %b) #0 {
 ; CHECK-LABEL: for function 'temporal_uniform_indivloop':
 ; CHECK-NOT: DIVERGENT: %uni.
 ; CHECK-NOT: DIVERGENT: br i1 %uni.
 
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %uni.cond = icmp slt i32 %a, 0
   br label %G
 
 G:
   br label %H
 
 H:
   %uni.merge.h = phi i32 [ 0, %G ], [ %uni.inc, %H ]
   %uni.inc = add i32 %uni.merge.h, 1
-  br i1 %uni.cond, label %X, label %H ; divergent branch
+  br i1 %uni.cond, label %X, label %H
 
 X:
   %uni.user = add i32 %uni.inc, 5
   %div.exity = icmp slt i32 %tid, 0
 ; CHECK: DIVERGENT: %div.exity =
   br i1 %div.exity, label %G, label %Y
 ; CHECK: DIVERGENT: br i1 %div.exity,
 
H:
   br i1 %div.exitx, label %X, label %H ; divergent branch
 ; CHECK: DIVERGENT: %div.exitx =
 ; CHECK: DIVERGENT: br i1 %div.exitx,
 
 X:
   br label %G
 
 G:
+; C HECK: DIVERGENT: %div.user =
   %div.user = add i32 %uni.inc, 5
   br i1 %uni.cond, label %G, label %Y
 ; CHECK: DIVERGENT: %div.user =
 
 Y:
   ret void
 }
 
-; temporal-divergent use of value carried by divergent loop, user is inside sibling loop, defs and use are carried by a uniform loop
+; temporal-divergent use of value carried by divergent loop, user is inside
+; sibling loop, defs and use are carried by a uniform loop
 define amdgpu_kernel void @temporal_diverge_loopuser_nested(i32 %n, i32 %a, i32 %b) #0 {
 ; CHECK-LABEL: for function 'temporal_diverge_loopuser_nested':
 ; CHECK-NOT: DIVERGENT: %uni.
 ; CHECK-NOT: DIVERGENT: br i1 %uni.
 
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %uni.cond = icmp slt i32 %a, 0