Index: llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/trunk/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4719,6 +4719,8 @@ // Holds all interleaved store groups temporarily. SmallSetVector StoreGroups; + // Holds all interleaved load groups temporarily. + SmallSetVector LoadGroups; // Search the load-load/write-write pair B-A in bottom-up order and try to // insert B into the interleave group of A according to 3 rules: @@ -4746,6 +4748,8 @@ if (A->mayWriteToMemory()) StoreGroups.insert(Group); + else + LoadGroups.insert(Group); for (auto II = std::next(I); II != E; ++II) { Instruction *B = II->first; @@ -4793,6 +4797,12 @@ for (InterleaveGroup *Group : StoreGroups) if (Group->getNumMembers() != Group->getFactor()) releaseGroup(Group); + + // Remove interleaved load groups that don't have the first and last member. + // This guarantees that we won't do speculative out of bounds loads. + for (InterleaveGroup *Group : LoadGroups) + if (!Group->getMember(0) || !Group->getMember(Group->getFactor() - 1)) + releaseGroup(Group); } LoopVectorizationCostModel::VectorizationFactor Index: llvm/trunk/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll =================================================================== --- llvm/trunk/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll +++ llvm/trunk/test/Transforms/LoopVectorize/PowerPC/stride-vectorization.ll @@ -16,9 +16,15 @@ for.body: ; preds = %for.body, %entry %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] %0 = shl nsw i64 %indvars.iv, 1 + %odd.idx = add nsw i64 %0, 1 + %arrayidx = getelementptr inbounds double, double* %b, i64 %0 + %arrayidx.odd = getelementptr inbounds double, double* %b, i64 %odd.idx + %1 = load double, double* %arrayidx, align 8 - %add = fadd double %1, 1.000000e+00 + %2 = load double, double* %arrayidx.odd, align 8 + + %add = fadd double %1, %2 %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv store double %add, double* %arrayidx2, align 8 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 Index: llvm/trunk/test/Transforms/LoopVectorize/interleaved-accesses.ll =================================================================== --- llvm/trunk/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ llvm/trunk/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -292,10 +292,8 @@ ; } ; CHECK-LABEL: @even_load( -; CHECK: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4 -; CHECK: %strided.vec = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> -; CHECK-NOT: shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> -; CHECK: shl nsw <4 x i32> %strided.vec, +; CHECK-NOT: %wide.vec = load <8 x i32>, <8 x i32>* %{{.*}}, align 4 +; CHECK-NOT: %strided.vec = shufflevector <8 x i32> %wide.vec, <8 x i32> undef, <4 x i32> define void @even_load(i32* noalias nocapture readonly %A, i32* noalias nocapture %B) { entry: