[GVN] Simple GVNHoist - limits

[chill]

Summary

This patch adds a parameter limiting how deep in a block we go looking
for instructions to hoist.

It also prevents certain instructions from initiating the hoist of a
dependency chain with the goal to not separate them from their
users. Such instructions are:

• GEPs, with the intent to keep them together with the (comparatively harder to hoist) loads/stores

• binary ops, which are converted to boolean by comparing them to zero: often the result of comparison to zero is (or can be chosen to be) a side effect from the original computation

• conditions to br and select instructions

Nevertheless, such instructions can be hoisted, if the process starts
from their users.

[junparser]

Would it better use some parameter from GVNHoist? Also I do think maybe the default value is too large.

[chill]

In D111418#3054542, @junparser wrote:

Would it better use some parameter from GVNHoist? Also I do think maybe the default value is too large.

I wanted to be independent from GVNHoist.cpp, e.g. choose different defaults. Indeed, defaults may well be large, I'm going to look
at various benchmarks and defaults' effect on them, and maybe change the defaults.

[junparser]

LGTM

[chill]

• Removed the maximum chain length limit as the current implementation does not iterate and does not need renumbering as the original GVNHoist
• Spent some effort to not separate certain instructions from their users (that's an outcome from some tuning/regression fixing with various benchmarks).

llvm/lib/Transforms/Scalar/GVN.cpp

 static cl::opt<uint32_t> MaxBBSpeculations(
     "gvn-max-block-speculations", cl::Hidden, cl::init(600), cl::ZeroOrMore,
     cl::desc("Max number of blocks we're willing to speculate on (and recurse "
              "into) when deducing if a value is fully available or not in GVN "
              "(default = 600)"));
 
 static cl::opt<bool> GVNEnableSimpleGVNHoist("enable-simple-gvn-hoist",
                                              cl::init(true));
+static cl::opt<uint32_t> MaxBlockDepth(
+    "simple-gvn-hoist-max-block-depth", cl::Hidden, cl::init(100),
+    cl::desc("Limit hoist candidates to the first N instructions in a block"));
 
 struct llvm::GVN::Expression {
   uint32_t opcode;
   bool commutative = false;
   Type *type = nullptr;
   SmallVector<uint32_t, 4> varargs;
 
   Expression(uint32_t o = ~2U) : opcode(o) {}
if (const auto *SA = dyn_cast<StoreInst>(&A)) {
     return SA->isVolatile() == SB->isVolatile() &&
            SA->getOrdering() == SB->getOrdering();
   }
 
   return true;
 }
 
 void GVN::collectHoistCandidates(BasicBlock *BB) {
+  uint32_t Depth = 0;
   bool HasHoistBarrier = false;
   for (Instruction &I : *BB) {
+    if (++Depth > MaxBlockDepth)
+      break;
     if (I.isTerminator())
       break;
     if (isa<PHINode>(I))
       continue;
 
     if (!HasHoistBarrier || isSafeToSpeculativelyExecute(&I))
       HoistPairs[VN.lookupOrAdd(&I)].first = &I;
 
     if ((HasHoistBarrier |= isHoistBarrier(I)))
       LLVM_DEBUG(dbgs() << "Simple GVNHoist: reached hoist barrier" << I
                         << '\n';);
   }
 }
 
 void GVN::matchHoistCandidates(BasicBlock *BB) {
+  uint32_t Depth = 0;
   bool HasHoistBarrier = false;
   for (Instruction &I : *BB) {
+    if (++Depth > MaxBlockDepth)
+      break;
     if (I.isTerminator())
       break;
     if (isa<PHINode>(I))
       continue;
 
     if (!HasHoistBarrier || isSafeToSpeculativelyExecute(&I)) {
       uint32_t N = VN.lookupOrAdd(&I);
       auto It = HoistPairs.find(N);
std::pair<bool, bool> GVN::hoistPair(BasicBlock *DestBB, BasicBlock *ThenBB,
   ThenI->moveBefore(DestBB->getTerminator());
   if (MemoryUseOrDef *MA = MSSAU->getMemorySSA()->getMemoryAccess(ThenI))
     MSSAU->moveToPlace(MA, DestBB, MemorySSA::InsertionPlace::BeforeTerminator);
   replaceInstruction(ElseI, ThenI);
 
   return {true, false};
 }
 
+// Determine if an instruction should be used to initiate hoisting a
+// dependency chain. The aim is to avoid separating instructions, for which it's
+// (heuristically) considered better to keep them together, as it's common that
+// they can be fused in some way. An instruction, which is denied hoisting by
+// this function can still be hoisted if it appears as a dependency (e.g
+// operand) of another hoisted instruction.
+static bool shouldNotInitiateHoisting(const Instruction *I) {
+  // Don't separate GEP's form their loads/stores.
+  if (isa<GetElementPtrInst>(I))
+    return true;
+  const bool IsBinop = isa<BinaryOperator>(I);
+  for (const User *U : I->users()) {
+    // Don't separate conditions from `br` or `select`.
+    if ((isa<BranchInst>(U) || isa<SelectInst>(U)) && U->getOperand(0) == I)
+      return true;
+    // Don't separate a value from converting that value to a boolean by
+    // comparing it to zero.
+    if (!IsBinop)
+      continue;
+    const auto *ICmp = dyn_cast<ICmpInst>(U);
+    if (ICmp == nullptr || (ICmp->getPredicate() != CmpInst::ICMP_EQ &&
+                            ICmp->getPredicate() != CmpInst::ICMP_NE))
+      continue;
+    const auto *Zero = dyn_cast<ConstantInt>(ICmp->getOperand(1));
+    if (Zero != nullptr && Zero->isZero())
+      return true;
+  }
+  return false;
+}
+
 // Perform trivial hoisting of values from two blocks to their common
 // predecessor.
 bool GVN::performHoist(Function &F) {
   LLVM_DEBUG(dbgs() << "Simple GVNHoist: running on function " << F.getName()
                     << '\n';);
   bool Change = false;
   ReversePostOrderTraversal<Function *> RPOT(&F);
   for (BasicBlock *BB : RPOT) {
for (BasicBlock *BB : RPOT) {
 
     HoistPairs.clear();
     collectHoistCandidates(Then);
     matchHoistCandidates(Else);
 
     // Hoist matched pairs.
     for (const auto &P : HoistPairs) {
       bool LocalChange, _;
-      std::tie(LocalChange, _) =
-          hoistPair(BB, Then, Else, P.second.first, P.second.second);
+      Instruction *ThenI = P.second.first, *ElseI = P.second.second;
+      if (shouldNotInitiateHoisting(ThenI))
+        continue;
+      std::tie(LocalChange, _) = hoistPair(BB, Then, Else, ThenI, ElseI);
       Change |= LocalChange;
     }
   }
 
   return Change;
 }
 
 class llvm::gvn::GVNLegacyPass : public FunctionPass {

llvm/test/Transforms/GVN/simple-gvn-hoist-limits.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S --passes=gvn -simple-gvn-hoist-max-block-depth=3 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH3
+; RUN: opt -S --passes=gvn -simple-gvn-hoist-max-block-depth=4 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH4
+; RUN: opt -S --passes=gvn -simple-gvn-hoist-max-block-depth=5 %s -o - | FileCheck %s --check-prefix=MAX-DEPTH5
+
+; Check effect of limiting the how deep in a block we look for
+; instructions to hoist; At max depth 3 we shouldn't hoist
+; anything to the `entry` block, at max depth 4 we should hoist
+; the first `add` and at max depth 5 the second `add`.
+
+; In any case we shouldn't hoist the `and` and the `icmp`, in order
+; to not separate them from the `br`
+
+define i32 @f(i1 %c, i32 %a, i32 %b, i32 %d) {
+; MAX-DEPTH3-LABEL: @f(
+; MAX-DEPTH3-NEXT:  entry:
+; MAX-DEPTH3-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH3:       if.then:
+; MAX-DEPTH3-NEXT:    [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH3-NEXT:    [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH3-NEXT:    [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH3-NEXT:    [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH3-NEXT:    [[S0:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH3-NEXT:    br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH3:       if.then1:
+; MAX-DEPTH3-NEXT:    [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH3-NEXT:    br label [[EXIT:%.*]]
+; MAX-DEPTH3:       if.else1:
+; MAX-DEPTH3-NEXT:    [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH3-NEXT:    br label [[EXIT]]
+; MAX-DEPTH3:       if.else:
+; MAX-DEPTH3-NEXT:    [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH3-NEXT:    [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH3-NEXT:    [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH3-NEXT:    [[D1:%.*]] = add i32 [[D]], 1
+; MAX-DEPTH3-NEXT:    [[S1:%.*]] = add i32 [[D1]], [[B]]
+; MAX-DEPTH3-NEXT:    br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH3:       if.then2:
+; MAX-DEPTH3-NEXT:    [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH3-NEXT:    br label [[EXIT]]
+; MAX-DEPTH3:       if.else2:
+; MAX-DEPTH3-NEXT:    [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH3-NEXT:    br label [[EXIT]]
+; MAX-DEPTH3:       exit:
+; MAX-DEPTH3-NEXT:    [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH3-NEXT:    [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH3-NEXT:    [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH3-NEXT:    ret i32 [[R]]
+;
+; MAX-DEPTH4-LABEL: @f(
+; MAX-DEPTH4-NEXT:  entry:
+; MAX-DEPTH4-NEXT:    [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH4-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH4:       if.then:
+; MAX-DEPTH4-NEXT:    [[R0:%.*]] = add i32 [[B:%.*]], 1
+; MAX-DEPTH4-NEXT:    [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH4-NEXT:    [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH4-NEXT:    [[S0:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH4-NEXT:    br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH4:       if.then1:
+; MAX-DEPTH4-NEXT:    [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH4-NEXT:    br label [[EXIT:%.*]]
+; MAX-DEPTH4:       if.else1:
+; MAX-DEPTH4-NEXT:    [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH4-NEXT:    br label [[EXIT]]
+; MAX-DEPTH4:       if.else:
+; MAX-DEPTH4-NEXT:    [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH4-NEXT:    [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH4-NEXT:    [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH4-NEXT:    [[S1:%.*]] = add i32 [[D0]], [[B]]
+; MAX-DEPTH4-NEXT:    br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH4:       if.then2:
+; MAX-DEPTH4-NEXT:    [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH4-NEXT:    br label [[EXIT]]
+; MAX-DEPTH4:       if.else2:
+; MAX-DEPTH4-NEXT:    [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH4-NEXT:    br label [[EXIT]]
+; MAX-DEPTH4:       exit:
+; MAX-DEPTH4-NEXT:    [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH4-NEXT:    [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S1]], [[IF_THEN2]] ], [ [[S1]], [[IF_ELSE2]] ]
+; MAX-DEPTH4-NEXT:    [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH4-NEXT:    ret i32 [[R]]
+;
+; MAX-DEPTH5-LABEL: @f(
+; MAX-DEPTH5-NEXT:  entry:
+; MAX-DEPTH5-NEXT:    [[D0:%.*]] = add i32 [[D:%.*]], 1
+; MAX-DEPTH5-NEXT:    [[S0:%.*]] = add i32 [[D0]], [[B:%.*]]
+; MAX-DEPTH5-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MAX-DEPTH5:       if.then:
+; MAX-DEPTH5-NEXT:    [[R0:%.*]] = add i32 [[B]], 1
+; MAX-DEPTH5-NEXT:    [[AND0:%.*]] = and i32 [[A:%.*]], 1
+; MAX-DEPTH5-NEXT:    [[TOBOOL_AND0:%.*]] = icmp eq i32 [[AND0]], 0
+; MAX-DEPTH5-NEXT:    br i1 [[TOBOOL_AND0]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]]
+; MAX-DEPTH5:       if.then1:
+; MAX-DEPTH5-NEXT:    [[OR0:%.*]] = or i32 [[R0]], 1
+; MAX-DEPTH5-NEXT:    br label [[EXIT:%.*]]
+; MAX-DEPTH5:       if.else1:
+; MAX-DEPTH5-NEXT:    [[OR1:%.*]] = or i32 [[R0]], 2
+; MAX-DEPTH5-NEXT:    br label [[EXIT]]
+; MAX-DEPTH5:       if.else:
+; MAX-DEPTH5-NEXT:    [[R1:%.*]] = add i32 [[B]], 2
+; MAX-DEPTH5-NEXT:    [[AND1:%.*]] = and i32 [[A]], 1
+; MAX-DEPTH5-NEXT:    [[TOBOOL_AND1:%.*]] = icmp eq i32 [[AND1]], 0
+; MAX-DEPTH5-NEXT:    br i1 [[TOBOOL_AND1]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]]
+; MAX-DEPTH5:       if.then2:
+; MAX-DEPTH5-NEXT:    [[OR2:%.*]] = or i32 [[R1]], 4
+; MAX-DEPTH5-NEXT:    br label [[EXIT]]
+; MAX-DEPTH5:       if.else2:
+; MAX-DEPTH5-NEXT:    [[OR3:%.*]] = or i32 [[R1]], 8
+; MAX-DEPTH5-NEXT:    br label [[EXIT]]
+; MAX-DEPTH5:       exit:
+; MAX-DEPTH5-NEXT:    [[OR:%.*]] = phi i32 [ [[OR0]], [[IF_THEN1]] ], [ [[OR1]], [[IF_ELSE1]] ], [ [[OR2]], [[IF_THEN2]] ], [ [[OR3]], [[IF_ELSE2]] ]
+; MAX-DEPTH5-NEXT:    [[S:%.*]] = phi i32 [ [[S0]], [[IF_THEN1]] ], [ [[S0]], [[IF_ELSE1]] ], [ [[S0]], [[IF_THEN2]] ], [ [[S0]], [[IF_ELSE2]] ]
+; MAX-DEPTH5-NEXT:    [[R:%.*]] = add i32 [[OR]], [[S]]
+; MAX-DEPTH5-NEXT:    ret i32 [[R]]
+;
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %r0 = add i32 %b, 1
+  %and0 = and i32 %a, 1
+  %tobool.and0 = icmp eq i32 %and0, 0
+  %d0 = add i32 %d, 1
+  %s0 = add i32 %d0, %b
+  br i1 %tobool.and0, label %if.then1, label %if.else1
+if.then1:
+  %or0 = or i32 %r0, 1
+  br label %exit
+if.else1:
+  %or1 = or i32 %r0, 2
+  br label %exit
+
+if.else:
+  %r1 = add i32 %b, 2
+  %and1 = and i32 %a, 1
+  %tobool.and1 = icmp eq i32 %and1, 0
+  %d1 = add i32 %d, 1
+  %s1 = add i32 %d1, %b
+  br i1 %tobool.and1, label %if.then2, label %if.else2
+if.then2:
+  %or2 = or i32 %r1, 4
+  br label %exit
+if.else2:
+  %or3 = or i32 %r1, 8
+  br label %exit
+exit:
+  %or = phi i32 [%or0, %if.then1], [%or1, %if.else1], [%or2, %if.then2], [%or3, %if.else2]
+  %s = phi i32 [%s0, %if.then1], [%s0, %if.else1], [%s1, %if.then2], [%s1, %if.else2]
+  %r = add i32 %or, %s
+  ret i32 %r
+}

llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll

 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4, !alias.scope !8
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 20, i32 20, i32 20, i32 20>
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>*
-; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4, !alias.scope !11
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul <4 x float> [[WIDE_LOAD14]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP5]] to <4 x float>*
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4, !alias.scope !11
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <4 x float> [[WIDE_LOAD14]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[TMP8]] to <4 x float>*
 ; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <4 x float>, <4 x float>* [[TMP9]], align 4, !alias.scope !13, !noalias !15
-; CHECK-NEXT:    [[TMP10:%.*]] = fadd <4 x float> [[TMP8]], [[WIDE_LOAD15]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[TMP8]], <4 x float> [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP5]] to <4 x float>*
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd <4 x float> [[TMP7]], [[WIDE_LOAD15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[TMP7]], <4 x float> [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast float* [[TMP8]] to <4 x float>*
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], <4 x float>* [[TMP11]], align 4, !alias.scope !13, !noalias !15
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       loop.body:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[C_GEP:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[IV1]]
 ; CHECK-NEXT:    [[C_LV:%.*]] = load i32, i32* [[C_GEP]], align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[C_LV]], 20
-; CHECK-NEXT:    [[B_GEP_0:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[IV1]]
 ; CHECK-NEXT:    [[A_GEP_0:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[IV1]]
 ; CHECK-NEXT:    [[A_LV_0:%.*]] = load float, float* [[A_GEP_0]], align 4
 ; CHECK-NEXT:    [[MUL2_I81_I:%.*]] = fmul float [[A_LV_0]], [[X]]
+; CHECK-NEXT:    [[B_GEP_0:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[IV1]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP_LATCH]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[B_LV:%.*]] = load float, float* [[B_GEP_0]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[MUL2_I81_I]], [[B_LV]]
 ; CHECK-NEXT:    br label [[LOOP_LATCH]]
 ; CHECK:       loop.latch:
-; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi float [ [[ADD]], [[ELSE]] ], [ [[MUL2_I81_I]], [[LOOP_BODY]] ]
-; CHECK-NEXT:    store float [[STOREMERGE]], float* [[B_GEP_0]], align 4
+; CHECK-NEXT:    [[ADD_SINK:%.*]] = phi float [ [[ADD]], [[ELSE]] ], [ [[MUL2_I81_I]], [[LOOP_BODY]] ]
+; CHECK-NEXT:    store float [[ADD_SINK]], float* [[B_GEP_0]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; CHECK-NEXT:    [[CMP_0:%.*]] = icmp ult i64 [[IV1]], 9999
 ; CHECK-NEXT:    br i1 [[CMP_0]], label [[LOOP_BODY]], label [[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop.header