[SimpleLoopUnswitch] Enable freezing of conditions by default.

[fhahn]

Summary

This fixes a series of mis-compiles by SimpleLoopUnswitch.

My measurements showed no performance regression with -O3 on AArch64
in SPEC2006, SPEC2017 and a set of internal benchmarks.

Fixes #50387, #50430

Depends on D124251.

[nikic]

LGTM

[aqjune]

Thank you for your work..! The diff looks fine.

[aqjune]

A quick question: should LoopUnswitch.cpp be fixed as well? It seems the pass is not maintained though. Will SimpleLoopUnswitch supercede the LoopUnswitch pass?

[nikic]

I believe LoopUnswitch is only used in the legacy PM, so we should probably just delete the pass.

[fhahn]

In D124252#3469719, @nikic wrote:

I believe LoopUnswitch is only used in the legacy PM, so we should probably just delete the pass.

Yes, the pass should effectively be dead code with the move to the new PM and I don't think we should spend any time on updating it. I'll put up a patch to remove it.

[aqjune]

It sounds great - then after the removal of the old pass I will make a patch that removes https://github.com/llvm/llvm-project/blob/main/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp#L2508 as well.

[fhahn]

In D124252#3469849, @fhahn wrote:

In D124252#3469719, @nikic wrote:

I believe LoopUnswitch is only used in the legacy PM, so we should probably just delete the pass.

Yes, the pass should effectively be dead code with the move to the new PM and I don't think we should spend any time on updating it. I'll put up a patch to remove it.

Patch is available: D124376

llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp

static cl::opt<bool> DropNonTrivialImplicitNullChecks(
     cl::desc("If enabled, drop make.implicit metadata in unswitched implicit "
              "null checks to save time analyzing if we can keep it."));
 static cl::opt<unsigned>
     MSSAThreshold("simple-loop-unswitch-memoryssa-threshold",
                   cl::desc("Max number of memory uses to explore during "
                            "partial unswitching analysis"),
                   cl::init(100), cl::Hidden);
 static cl::opt<bool> FreezeLoopUnswitchCond(
-    "freeze-loop-unswitch-cond", cl::init(false), cl::Hidden,
+    "freeze-loop-unswitch-cond", cl::init(true), cl::Hidden,
     cl::desc("If enabled, the freeze instruction will be added to condition "
              "of loop unswitch to prevent miscompilation."));
 
 /// Collect all of the loop invariant input values transitively used by the
 /// homogeneous instruction graph from a given root.
 ///
 /// This essentially walks from a root recursively through loop variant operands
 /// which have the exact same opcode and finds all inputs which are loop

llvm/test/Transforms/SimpleLoopUnswitch/ARM/nontrivial-unswitch-cost.ll

 
 loop_exit:
   ret void
 }
 
 define void @test_unswitch(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_BEGIN_US:%.*]]
 ; CHECK:       loop_begin.us:
 ; CHECK-NEXT:    call void @x()
 ; CHECK-NEXT:    br label [[LOOP_A_US:%.*]]
 ; CHECK:       loop_a.us:
 ; CHECK-NEXT:    call void @a()
 ; CHECK-NEXT:    br label [[LOOP_LATCH_US:%.*]]

 
 loop_exit:
   ret void
 }
 
 define void @test_unswitch_non_dup_code(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_non_dup_code(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_BEGIN_US:%.*]]
 ; CHECK:       loop_begin.us:
 ; CHECK-NEXT:    call void @x()
 ; CHECK-NEXT:    br label [[LOOP_A_US:%.*]]
 ; CHECK:       loop_a.us:
 ; CHECK-NEXT:    call void @a()
 ; CHECK-NEXT:    call void @a()

 
 loop_exit:
   ret void
 }
 
 define void @test_unswitch_non_dup_code_in_cfg(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_non_dup_code_in_cfg(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_BEGIN_US:%.*]]
 ; CHECK:       loop_begin.us:
 ; CHECK-NEXT:    call void @x()
 ; CHECK-NEXT:    br label [[LOOP_A_US:%.*]]
 ; CHECK:       loop_a.us:
 ; CHECK-NEXT:    [[V1_US:%.*]] = load i1, i1* [[PTR:%.*]], align 1
 ; CHECK-NEXT:    br i1 [[V1_US]], label [[LOOP_A_A_US:%.*]], label [[LOOP_A_B_US:%.*]]

 
 loop_exit:
   ret void
 }
 
 define void @test_unswitch_large_exit(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_large_exit(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_BEGIN_US:%.*]]
 ; CHECK:       loop_begin.us:
 ; CHECK-NEXT:    call void @x()
 ; CHECK-NEXT:    br label [[LOOP_A_US:%.*]]
 ; CHECK:       loop_a.us:
 ; CHECK-NEXT:    call void @a()
 ; CHECK-NEXT:    br label [[LOOP_LATCH_US:%.*]]
loop_exit:
   call void @x()
   call void @x()
   ret void
 }
 
 define void @test_unswitch_dedicated_exiting(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_dedicated_exiting(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_BEGIN_US:%.*]]
 ; CHECK:       loop_begin.us:
 ; CHECK-NEXT:    call void @x()
 ; CHECK-NEXT:    br label [[LOOP_A_US:%.*]]
 ; CHECK:       loop_a.us:
 ; CHECK-NEXT:    call void @a()
 ; CHECK-NEXT:    br label [[LOOP_LATCH_US:%.*]]

llvm/test/Transforms/SimpleLoopUnswitch/guards.ll

 
 exit:
   ret void
 }
 
 define void @test_conditional_guards(i1 %cond, i32 %N) {
 ; CHECK-LABEL: @test_conditional_guards(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 [[COND:%.*]]
+; CHECK-NEXT:    br i1 [[FROZEN]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_US:%.*]]
 ; CHECK:       loop.us:
 ; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[IV_NEXT_US:%.*]], [[BACKEDGE_US:%.*]] ]
 ; CHECK-NEXT:    [[CONDITION_US:%.*]] = icmp eq i32 [[IV_US]], 123
 ; CHECK-NEXT:    br i1 [[CONDITION_US]], label [[GUARD_US:%.*]], label [[BACKEDGE_US]]
 ; CHECK:       guard.us:
 ; CHECK-NEXT:    br label [[GUARDED_US:%.*]]

llvm/test/Transforms/SimpleLoopUnswitch/implicit-null-checks.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -enable-nontrivial-unswitch=true -simple-loop-unswitch -S < %s | FileCheck %s
 ; RUN: opt -enable-nontrivial-unswitch=true -passes='loop(simple-loop-unswitch),verify<loops>' -S < %s | FileCheck %s
 
 declare void @may_exit()
 declare void @throw_npe()
 
 ; It is illegal to preserve make_implicit notion of the condition being
 ; unswitched because we may exit loop before we reach the condition, so
 ; there is no guarantee that following implicit branch always means getting
 ; to throw_npe block.
 define i32 @test_should_drop_make_implicit(i32* %p1, i32* %p2) {
 ; CHECK-LABEL: @test_should_drop_make_implicit(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
-; CHECK-NOT:     !make.implicit
-; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[NULL_CHECK_FR:%.*]] = freeze i1 [[NULL_CHECK]]
+; CHECK-NEXT:    br i1 [[NULL_CHECK_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_US:%.*]]
 ; CHECK:       loop.us:
 ; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ]
 ; CHECK-NEXT:    [[X_US:%.*]] = load i32, i32* [[P1:%.*]], align 4
 ; CHECK-NEXT:    [[SIDE_EXIT_COND_US:%.*]] = icmp eq i32 [[X_US]], 0
 ; CHECK-NEXT:    br i1 [[SIDE_EXIT_COND_US]], label [[SIDE_EXIT_SPLIT_US:%.*]], label [[NULL_CHECK_BLOCK_US:%.*]]
 ; CHECK:       null_check_block.us:
exit:
   ret i32 %x
 }
 
 ; Here make.implicit notion should be dropped because of exiting call.
 define i32 @test_should_drop_make_implicit_exiting_call(i32* %p1, i32* %p2) {
 ; CHECK-LABEL: @test_should_drop_make_implicit_exiting_call(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[NULL_CHECK:%.*]] = icmp eq i32* [[P2:%.*]], null
-; CHECK-NOT:     !make.implicit
-; CHECK-NEXT:    br i1 [[NULL_CHECK]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
+; CHECK-NEXT:    [[NULL_CHECK_FR:%.*]] = freeze i1 [[NULL_CHECK]]
+; CHECK-NEXT:    br i1 [[NULL_CHECK_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[LOOP_US:%.*]]
 ; CHECK:       loop.us:
 ; CHECK-NEXT:    [[IV_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ]
 ; CHECK-NEXT:    call void @may_exit()
 ; CHECK-NEXT:    [[X_US:%.*]] = load i32, i32* [[P1:%.*]], align 4
 ; CHECK-NEXT:    [[SIDE_EXIT_COND_US:%.*]] = icmp eq i32 [[X_US]], 0
 ; CHECK-NEXT:    br label [[THROW_NPE_SPLIT_US:%.*]]

llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-cost.ll

 
 ; Now check that the smaller formulation of 'loop_begin' does in fact unswitch
 ; with our low threshold.
 define void @test_unswitch(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 [[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   call void @x()
   br i1 %cond, label %loop_a, label %loop_b
 
 loop_a:
   call void @a()
   br label %loop_latch

 ; Check that even with large amounts of code on either side of the unswitched
 ; branch, if that code would be kept in only one of the unswitched clones it
 ; doesn't contribute to the cost.
 define void @test_unswitch_non_dup_code(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_non_dup_code(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 [[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   call void @x()
   br i1 %cond, label %loop_a, label %loop_b
 
 loop_a:
   call void @a()
   call void @a()

 
 ; Much like with non-duplicated code directly in the successor, we also won't
 ; duplicate even interesting CFGs.
 define void @test_unswitch_non_dup_code_in_cfg(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_non_dup_code_in_cfg(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 [[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   call void @x()
   br i1 %cond, label %loop_a, label %loop_b
 
 loop_a:
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %loop_a_a, label %loop_a_b

 ; Check that we still unswitch when the exit block contains lots of code, even
 ; though we do clone the exit block as part of unswitching. This should work
 ; because we should split the exit block before anything inside it.
 define void @test_unswitch_large_exit(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_large_exit(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 [[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   call void @x()
   br i1 %cond, label %loop_a, label %loop_b
 
 loop_a:
   call void @a()
   br label %loop_latch

 
 ; Check that we handle a dedicated exit edge unswitch which is still
 ; non-trivial and has lots of code in the exit.
 define void @test_unswitch_dedicated_exiting(i1* %ptr, i1 %cond) {
 ; CHECK-LABEL: @test_unswitch_dedicated_exiting(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 [[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   call void @x()
   br i1 %cond, label %loop_a, label %loop_b_exit
 
 loop_a:
   call void @a()
   br label %loop_latch

llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll

inner_loop_begin:
   %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
   %cond = load i1, i1* %cond.ptr
   %b = load i32, i32* %b.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_loop_begin:
 ; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
 ; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
 
 inner_inner_loop_begin:
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr
   br i1 %v2, label %loop_exit, label %inner_inner_loop_c
inner_loop_begin:
   %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
   %cond = load i1, i1* %cond.ptr
   %b = load i32, i32* %b.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_loop_begin:
 ; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
 ; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
 
 inner_inner_loop_begin:
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr
   br i1 %v2, label %loop_exit, label %inner_inner_loop_c
inner_loop_begin:
   %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
   %cond = load i1, i1* %cond.ptr
   %b = load i32, i32* %b.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_loop_begin:
 ; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
 ; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
 
 inner_inner_loop_begin:
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr
   br i1 %v2, label %inner_inner_loop_latch, label %inner_loop_exit
inner_loop_begin:
   %a.phi = phi i32 [ %a, %loop_begin ], [ %a2, %inner_inner_loop_exit ]
   %cond = load i1, i1* %cond.ptr
   %b = load i32, i32* %b.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_loop_begin:
 ; CHECK-NEXT:    %[[A_INNER_PHI:.*]] = phi i32 [ %[[A]], %loop_begin ], [ %[[A2:.*]], %inner_inner_loop_exit ]
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
 ; CHECK-NEXT:    %[[B:.*]] = load i32, i32* %b.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_begin.split.us, label %inner_loop_begin.split
 
 inner_inner_loop_begin:
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %inner_inner_loop_a, label %inner_inner_loop_b
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr
   br i1 %v2, label %inner_inner_loop_latch, label %inner_loop_exit

 }
 
 ; Test that requires re-forming dedicated exits for the cloned loop.
 define i32 @test10a(i1* %ptr, i1 %cond, i32* %a.ptr) {
 ; CHECK-LABEL: @test10a(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   %a = load i32, i32* %a.ptr
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %loop_a, label %loop_b
 
 loop_a:
   %v2 = load i1, i1* %ptr

 }
 
 ; Test that requires re-forming dedicated exits for the original loop.
 define i32 @test10b(i1* %ptr, i1 %cond, i32* %a.ptr) {
 ; CHECK-LABEL: @test10b(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   %a = load i32, i32* %a.ptr
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %loop_a, label %loop_b
 
 loop_a:
   %v2 = load i1, i1* %ptr

 ; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
 ; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %inner_loop_ph
 
 inner_loop_ph:
   %cond = load i1, i1* %cond.ptr
   br label %inner_loop_begin
 ; CHECK:       inner_loop_ph:
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
 
 inner_loop_begin:
   call void @sink1(i32 %b)
   %a = load i32, i32* %a.ptr
   br i1 %cond, label %loop_exit, label %inner_loop_a
 
 inner_loop_a:
   %v2 = load i1, i1* %ptr

 ; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
 ; CHECK-NEXT:    br i1 %[[V]], label %loop_latch, label %inner_loop_ph
 
 inner_loop_ph:
   %cond = load i1, i1* %cond.ptr
   br label %inner_loop_begin
 ; CHECK:       inner_loop_ph:
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_loop_ph.split.us, label %inner_loop_ph.split
 
 inner_loop_begin:
   call void @sink1(i32 %b)
   %a = load i32, i32* %a.ptr
   br i1 %cond, label %inner_loop_a, label %loop_exit
 
 inner_loop_a:
   %v2 = load i1, i1* %ptr

 ; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
 ; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_inner_loop_ph
 
 inner_inner_loop_ph:
   %cond = load i1, i1* %cond.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_inner_loop_ph:
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
 
 inner_inner_loop_begin:
   call void @sink1(i32 %b)
   %a = load i32, i32* %a.ptr
   br i1 %cond, label %inner_loop_exit, label %inner_inner_loop_a
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr

 ; CHECK-NEXT:    %[[V:.*]] = load i1, i1* %ptr
 ; CHECK-NEXT:    br i1 %[[V]], label %inner_loop_latch, label %inner_inner_loop_ph
 
 inner_inner_loop_ph:
   %cond = load i1, i1* %cond.ptr
   br label %inner_inner_loop_begin
 ; CHECK:       inner_inner_loop_ph:
 ; CHECK-NEXT:    %[[COND:.*]] = load i1, i1* %cond.ptr
-; CHECK-NEXT:    br i1 %[[COND]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %[[COND]]
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %inner_inner_loop_ph.split.us, label %inner_inner_loop_ph.split
 
 inner_inner_loop_begin:
   call void @sink1(i32 %b)
   %a = load i32, i32* %a.ptr
   br i1 %cond, label %inner_inner_loop_a, label %inner_loop_exit
 
 inner_inner_loop_a:
   %v2 = load i1, i1* %ptr

 ; exiting block that connects the inner loop to the cloned loop is not the header
 ; block. This ensures that we correctly handle interesting corner cases of
 ; traversing back to the header when establishing the cloned loop.
 define i32 @test13a(i1* %ptr, i1 %cond, i32* %a.ptr, i32* %b.ptr) {
 ; CHECK-LABEL: @test13a(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   %a = load i32, i32* %a.ptr
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %loop_a, label %loop_b
 
 loop_a:
   %v2 = load i1, i1* %ptr

 ; the header block. This ensures that we correctly handle interesting corner
 ; cases of traversing back to the header when re-establishing the original loop
 ; still exists after unswitching.
 define i32 @test13b(i1* %ptr, i1 %cond, i32* %a.ptr, i32* %b.ptr) {
 ; CHECK-LABEL: @test13b(
 entry:
   br label %loop_begin
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %cond
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %entry.split.us, label %entry.split
 
 loop_begin:
   %a = load i32, i32* %a.ptr
   %v1 = load i1, i1* %ptr
   br i1 %v1, label %loop_a, label %loop_b
 
 loop_a:
   %v2 = load i1, i1* %ptr

 ; predecessors of the unswitched loop header. That can allow us to find the loop
 ; through multiple different paths.
 define void @test21(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test21(
 bb:
   br label %bb3
 ; CHECK-NOT:     br i1 %a
 ;
-; CHECK:         br i1 %a, label %[[BB_SPLIT_US:.*]], label %[[BB_SPLIT:.*]]
+; CHECK:         %[[FROZEN:.+]] = freeze i1 %a
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %[[BB_SPLIT_US:.*]], label %[[BB_SPLIT:.*]]
 ;
 ; CHECK-NOT:     br i1 %a
 ; CHECK-NOT:     br i1 %b
 ;
 ; CHECK:       [[BB_SPLIT]]:
 ; CHECK:         br i1 %b
 ;
 ; CHECK-NOT:     br i1 %a

 ; skip whole subregions of the outer loop blocks but just because the header of
 ; the outer loop is also the preheader of an inner loop shouldn't confuse this
 ; walk.
 define void @test23(i1 %arg, i1* %ptr) {
 ; CHECK-LABEL: define void @test23(
 entry:
   br label %outer.header
 ; CHECK:       entry:
-; CHECK-NEXT:    br i1 %arg,
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %arg
+; CHECK-NEXT:    br i1 %[[FROZEN]],
 ;
 ; Just verify that we unswitched the correct bits. We should call `@f` twice in
 ; one unswitch and `@f` and then `@g` in the other.
 ; CHECK:         call void
 ; CHECK-SAME:              @f
 ; CHECK:         call void
 ; CHECK-SAME:              @f
 ;

 ; can introduce multiple edges to successors. These need lots of special case
 ; handling as they get collapsed in many cases (domtree, the unswitch itself)
 ; but not in all cases (the PHI node operands).
 define i32 @test29(i32 %arg) {
 ; CHECK-LABEL: @test29(
 entry:
   br label %header
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_C:.*]] [
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i32 %arg
+; CHECK-NEXT:    switch i32 %[[FROZEN]], label %[[ENTRY_SPLIT_C:.*]] [
 ; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
 ; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_A]]
 ; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_B:.*]]
 ; CHECK-NEXT:      i32 3, label %[[ENTRY_SPLIT_C]]
 ; CHECK-NEXT:    ]
 
 header:
   %tmp = call i32 @d()

 ; a loop exit edge as those can in some cases be special. Among other things,
 ; this includes an LCSSA phi with multiple entries despite being a dedicated
 ; exit block.
 define i32 @test30(i32 %arg) {
 ; CHECK-LABEL: define i32 @test30(
 entry:
   br label %header
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_EXIT:.*]] [
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i32 %arg
+; CHECK-NEXT:    switch i32 %[[FROZEN]], label %[[ENTRY_SPLIT_EXIT:.*]] [
 ; CHECK-NEXT:      i32 -1, label %[[ENTRY_SPLIT_EXIT]]
 ; CHECK-NEXT:      i32 0, label %[[ENTRY_SPLIT_A:.*]]
 ; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_B:.*]]
 ; CHECK-NEXT:      i32 2, label %[[ENTRY_SPLIT_B]]
 ; CHECK-NEXT:    ]
 
 header:
   %tmp = call i32 @d()

 ; CHECK:       a.header:
 ; CHECK-NEXT:    br label %b.header
 
 b.header:
   %v1 = call i1 @cond()
   br label %c.header
 ; CHECK:       b.header:
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[B_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
 ;
 ; CHECK:       [[C_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @c()
 ; CHECK-NEXT:    br label %[[B_LATCH_SPLIT_US:.*]]
 ;

 
 b.header:
   %x.b = load i32, i32* %ptr
   %v1 = call i1 @cond()
   br label %c.header
 ; CHECK:       b.header:
 ; CHECK-NEXT:    %x.b = load i32, i32* %ptr
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[B_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
 ;
 ; CHECK:       [[C_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @c()
 ; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
 ;

 
 b.header:
   %x.b = load i32, i32* %ptr
   %v1 = call i1 @cond()
   br label %c.header
 ; CHECK:       b.header:
 ; CHECK-NEXT:    %x.b = load i32, i32* %ptr
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[B_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
 ;
 ; CHECK:       [[C_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @c()
 ; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
 ;

 
 b.header:
   %x.b = load i32, i32* %ptr
   %v1 = call i1 @cond()
   br label %c.header
 ; CHECK:       b.header:
 ; CHECK-NEXT:    %x.b = load i32, i32* %ptr
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    %[[FROZEN:.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 %[[FROZEN]], label %[[B_HEADER_SPLIT_US:.*]], label %[[B_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[B_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
 ;
 ; CHECK:       [[C_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @c()
 ; CHECK-NEXT:    br label %[[B_LATCH_US:.*]]
 ;

 ; CHECK:       b.header:
 ; CHECK-NEXT:    br label %c.header
 
 c.header:
   %v1 = call i1 @cond()
   br label %d.header
 ; CHECK:       c.header:
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 [[FROZEN]], label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[C_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[D_HEADER_US:.*]]
 ;
 ; CHECK:       [[D_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @d()
 ; CHECK-NEXT:    br label %[[C_LATCH_US:.*]]
 ;

 
 c.header:
   %x.c = load i32, i32* %ptr
   %v1 = call i1 @cond()
   br label %d.header
 ; CHECK:       c.header:
 ; CHECK-NEXT:    %x.c = load i32, i32* %ptr
 ; CHECK-NEXT:    %v1 = call i1 @cond()
-; CHECK-NEXT:    br i1 %v1, label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i1 %v1
+; CHECK-NEXT:    br i1 [[FROZEN]], label %[[C_HEADER_SPLIT_US:.*]], label %[[C_HEADER_SPLIT:.*]]
 ;
 ; CHECK:       [[C_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[D_HEADER_US:.*]]
 ;
 ; CHECK:       [[D_HEADER_US]]:
 ; CHECK-NEXT:    call i32 @d()
 ; CHECK-NEXT:    br label %[[C_LATCH_US:.*]]
 ;

 
 b.header:
   %x.b = load i32, i32* %ptr
   %v1 = call i32 @cond.i32()
   br label %c.header
 ; CHECK:       b.header:
 ; CHECK-NEXT:    %x.b = load i32, i32* %ptr
 ; CHECK-NEXT:    %v1 = call i32 @cond.i32()
-; CHECK-NEXT:    switch i32 %v1, label %[[B_HEADER_SPLIT:.*]] [
+; CHECK-NEXT:    [[FROZEN:%.+]] = freeze i32 %v1
+; CHECK-NEXT:    switch i32 [[FROZEN]], label %[[B_HEADER_SPLIT:.*]] [
 ; CHECK-NEXT:      i32 1, label %[[B_HEADER_SPLIT_US:.*]]
 ; CHECK-NEXT:      i32 2, label %[[B_HEADER_SPLIT_US]]
 ; CHECK-NEXT:      i32 3, label %[[B_HEADER_SPLIT_US]]
 ; CHECK-NEXT:    ]
 ;
 ; CHECK:       [[B_HEADER_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[C_HEADER_US:.*]]
 ;