Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4572,6 +4572,15 @@ void InnerLoopVectorizer::vectorizeBlockInLoop(BasicBlock *BB, PhiVector *PV) { // For each instruction in the old loop. for (Instruction &I : *BB) { + + // Scalarize instructions that should remain scalar after vectorization. + if (!(isa(&I) || isa(&I) || + isa(&I)) && + Legal->isScalarAfterVectorization(&I)) { + scalarizeInstruction(&I); + continue; + } + switch (I.getOpcode()) { case Instruction::Br: // Nothing to do for PHIs and BR, since we already took care of the Index: test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll =================================================================== --- test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll +++ test/Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll @@ -1,5 +1,6 @@ ; RUN: opt < %s -loop-vectorize -S | FileCheck %s +; CHECK: vector.body: ; CHECK: fadd ; CHECK-NEXT: fadd ; CHECK-NEXT: fadd @@ -12,9 +13,8 @@ ; CHECK-NEXT: fadd ; CHECK-NEXT: fadd ; CHECK-NEXT: fadd -; CHECK-NEXT: = ; CHECK-NOT: fadd -; CHECK-SAME: > +; CHECK: middle.block target datalayout = "e-m:e-i64:64-n32:64" target triple = "powerpc64le-ibm-linux-gnu" Index: test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll =================================================================== --- test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll +++ test/Transforms/LoopVectorize/PowerPC/vsx-tsvc-s173.ll @@ -43,7 +43,7 @@ ; CHECK-LABEL: @s173 ; CHECK: load <4 x float>, <4 x float>* -; CHECK: add i64 %index, 16000 +; CHECK: add nsw i64 %index, 16000 ; CHECK: ret i32 0 } Index: test/Transforms/LoopVectorize/global_alias.ll =================================================================== --- test/Transforms/LoopVectorize/global_alias.ll +++ test/Transforms/LoopVectorize/global_alias.ll @@ -387,7 +387,7 @@ ; return Foo.A[a]; ; } ; CHECK-LABEL: define i32 @noAlias08( -; CHECK: sub <4 x i32> +; CHECK: load <4 x i32> ; CHECK: ret define i32 @noAlias08(i32 %a) #0 { @@ -439,7 +439,7 @@ ; return Foo.A[a]; ; } ; CHECK-LABEL: define i32 @noAlias09( -; CHECK: sub <4 x i32> +; CHECK: load <4 x i32> ; CHECK: ret define i32 @noAlias09(i32 %a) #0 { @@ -721,7 +721,7 @@ ; return Foo.A[a]; ; } ; CHECK-LABEL: define i32 @noAlias14( -; CHECK: sub <4 x i32> +; CHECK: load <4 x i32> ; CHECK: ret define i32 @noAlias14(i32 %a) #0 { Index: test/Transforms/LoopVectorize/induction_plus.ll =================================================================== --- test/Transforms/LoopVectorize/induction_plus.ll +++ test/Transforms/LoopVectorize/induction_plus.ll @@ -9,7 +9,9 @@ ;CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] ;CHECK: %vec.ind = phi <4 x i64> [ , %vector.ph ], [ %vec.ind.next, %vector.body ] ;CHECK: %vec.ind1 = phi <4 x i32> [ , %vector.ph ], [ %vec.ind.next2, %vector.body ] -;CHECK: add nsw <4 x i64> %vec.ind, +;CHECK: %[[T1:.+]] = add i64 %index, 0 +;CHECK: %[[T2:.+]] = add nsw i64 %[[T1]], 12 +;CHECK: getelementptr inbounds [1024 x i32], [1024 x i32]* @array, i64 0, i64 %[[T2]] ;CHECK: %vec.ind.next = add <4 x i64> %vec.ind, ;CHECK: %vec.ind.next2 = add <4 x i32> %vec.ind1, ;CHECK: ret i32