Index: llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp =================================================================== --- llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp +++ llvm/trunk/lib/Transforms/Utils/LoopUtils.cpp @@ -565,7 +565,8 @@ auto *I = Phi->user_back(); if (I->isCast() && (I->getParent() == Phi->getParent()) && I->hasOneUse() && DT->dominates(Previous, I->user_back())) { - SinkAfter[I] = Previous; + if (!DT->dominates(Previous, I)) // Otherwise we're good w/o sinking. + SinkAfter[I] = Previous; return true; } } Index: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7655,7 +7655,7 @@ // 2. Copy and widen instructions from the old loop into the new loop. // Move instructions to handle first-order recurrences. - DenseMap SinkAfter = Legal->getSinkAfter(); + DenseMap &SinkAfter = Legal->getSinkAfter(); for (auto &Entry : SinkAfter) { Entry.first->removeFromParent(); Entry.first->insertAfter(Entry.second); Index: llvm/trunk/test/Transforms/LoopVectorize/first-order-recurrence.ll =================================================================== --- llvm/trunk/test/Transforms/LoopVectorize/first-order-recurrence.ll +++ llvm/trunk/test/Transforms/LoopVectorize/first-order-recurrence.ll @@ -140,7 +140,10 @@ ; CHECK: vector.body: ; CHECK: %vector.recur = phi <4 x i16> [ %vector.recur.init, %vector.ph ], [ [[L1:%[a-zA-Z0-9.]+]], %vector.body ] ; CHECK: [[L1]] = load <4 x i16> -; CHECK: {{.*}} = shufflevector <4 x i16> %vector.recur, <4 x i16> [[L1]], <4 x i32> +; CHECK: [[SHUF:%[a-zA-Z0-9.]+]] = shufflevector <4 x i16> %vector.recur, <4 x i16> [[L1]], <4 x i32> +; Check also that the casts were not moved needlessly. +; CHECK: sitofp <4 x i16> [[L1]] to <4 x double> +; CHECK: sitofp <4 x i16> [[SHUF]] to <4 x double> ; CHECK: middle.block: ; CHECK: %vector.recur.extract = extractelement <4 x i16> [[L1]], i32 3 ; CHECK: scalar.ph: