Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -5002,15 +5002,10 @@ for (LoopBlocksDFS::RPOIterator I = DFS.beginRPO(), E = DFS.endRPO(); I != E; ++I) { BasicBlock *BB = *I; - bool IsPred = LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT); for (auto &I : *BB) { if (!isa(&I) && !isa(&I)) continue; - // FIXME: Currently we can't handle mixed accesses and predicated accesses - if (IsPred) - return; - AccessList.push_back(&I); } } @@ -5185,6 +5180,13 @@ break; } + // If instruction B is in a predicated block, we can't form an + // interleaved group with instruction A unless A and B are in the same + // block. + if (LoopAccessInfo::blockNeedsPredication(B->getParent(), TheLoop, DT) && + A->getParent() != B->getParent()) + continue; + // The index of B is the index of A plus the related index to A. int IndexB = Group->getIndex(A) + DistanceToA / static_cast(DesA.Size); Index: test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll =================================================================== --- /dev/null +++ test/Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll @@ -0,0 +1,107 @@ +; RUN: opt -S -loop-vectorize -instcombine -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses -vectorize-num-stores-pred=1 -enable-cond-stores-vec < %s | FileCheck %s + +target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" + +%pair = type { i64, i64 } + +; CHECK-LABEL: @load_gap_with_pred_store_0 +; +; CHECK: min.iters.checked +; CHECK: %n.mod.vf = and i64 %[[N:.+]], 1 +; CHECK: %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0 +; CHECK: %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf +; CHECK: %n.vec = sub i64 %[[N]], %[[R]] +; +; CHECK: vector.body: +; CHECK: %wide.vec = load <4 x i64>, <4 x i64>* %{{.*}} +; CHECK: %strided.vec = shufflevector <4 x i64> %wide.vec, <4 x i64> undef, <2 x i32> +; CHECK: %[[Result:.+]] = and <2 x i64> %strided.vec, %{{.*}} +; +; CHECK: pred.store.if +; CHECK: %[[X1:.+]] = extractelement <2 x i64> %[[Result]], i32 0 +; CHECK: store i64 %[[X1]], {{.*}} +; +; CHECK: pred.store.if +; CHECK: %[[X2:.+]] = extractelement <2 x i64> %[[Result]], i32 1 +; CHECK: store i64 %[[X2]], {{.*}} + +define void @load_gap_with_pred_store_0(%pair *%p, i64 %x, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] + %p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 + %0 = load i64, i64* %p.1, align 8 + %1 = and i64 %0, %x + %2 = icmp eq i64 %1, %x + br i1 %2, label %if.then, label %if.merge + +if.then: + store i64 %1, i64* %p.1, align 8 + br label %if.merge + +if.merge: + %i.next = add nuw nsw i64 %i, 1 + %cond = icmp slt i64 %i.next, %n + br i1 %cond, label %for.body, label %for.end + +for.end: + ret void +} + +; CHECK-LABEL: @load_gap_with_pred_store_1 +; +; CHECK: min.iters.checked +; CHECK: %n.mod.vf = and i64 %[[N:.+]], 1 +; CHECK: %[[IsZero:[a-zA-Z0-9]+]] = icmp eq i64 %n.mod.vf, 0 +; CHECK: %[[R:.+]] = select i1 %[[IsZero]], i64 2, i64 %n.mod.vf +; CHECK: %n.vec = sub i64 %[[N]], %[[R]] +; +; CHECK: vector.body: +; CHECK: %[[L1:.+]] = load <4 x i64>, <4 x i64>* %{{.*}} +; CHECK: %strided.vec = shufflevector <4 x i64> %[[L1]], <4 x i64> undef, <2 x i32> +; CHECK: %[[Result:.+]] = and <2 x i64> %strided.vec, %{{.*}} +; +; CHECK: pred.store.if +; CHECK: %[[X1:.+]] = extractelement <2 x i64> %[[Result]], i32 0 +; CHECK: store i64 %[[X1]], {{.*}} +; +; CHECK: pred.store.if +; CHECK: %[[X2:.+]] = extractelement <2 x i64> %[[Result]], i32 1 +; CHECK: store i64 %[[X2]], {{.*}} +; +; CHECK: pred.store.continue +; CHECK: %[[L2:.+]] = load <4 x i64>, <4 x i64>* {{.*}} +; CHECK: %[[X3:.+]] = extractelement <4 x i64> %[[L2]], i32 0 +; CHECK: store i64 %[[X3]], {{.*}} +; CHECK: %[[X4:.+]] = extractelement <4 x i64> %[[L2]], i32 2 +; CHECK: store i64 %[[X4]], {{.*}} + +define void @load_gap_with_pred_store_1(%pair *%p, i64 %x, i64 %n) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ %i.next, %if.merge ], [ 0, %entry ] + %p.0 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 0 + %p.1 = getelementptr inbounds %pair, %pair* %p, i64 %i, i32 1 + %0 = load i64, i64* %p.1, align 8 + %1 = and i64 %0, %x + %2 = icmp eq i64 %1, %x + br i1 %2, label %if.then, label %if.merge + +if.then: + store i64 %1, i64* %p.0, align 8 + br label %if.merge + +if.merge: + %3 = load i64, i64* %p.0, align 8 + store i64 %3, i64 *%p.1, align 8 + %i.next = add nuw nsw i64 %i, 1 + %cond = icmp slt i64 %i.next, %n + br i1 %cond, label %for.body, label %for.end + +for.end: + ret void +}