diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2830,6 +2830,8 @@ // Keeps the reordered operands to avoid code duplication. SmallVector OperandsVec; for (unsigned I = 0, E = PH->getNumIncomingValues(); I < E; ++I) { + if (!DT->isReachableFromEntry(PH->getIncomingBlock(I))) + continue; ValueList Operands; // Prepare the operand vector. for (Value *V : VL) @@ -4301,8 +4303,15 @@ continue; OrderedScalars.push_back(Inst); } - llvm::stable_sort(OrderedScalars, [this](Instruction *A, Instruction *B) { - return DT->dominates(B, A); + llvm::sort(OrderedScalars, [&](Instruction *A, Instruction *B) { + auto *NodeA = DT->getNode(A->getParent()); + auto *NodeB = DT->getNode(B->getParent()); + assert(NodeA && "Should only process reachable instructions"); + assert(NodeB && "Should only process reachable instructions"); + assert((NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn())); + if (NodeA != NodeB) + return NodeA->getDFSNumIn() < NodeB->getDFSNumIn(); + return B->comesBefore(A); }); for (Instruction *Inst : OrderedScalars) { @@ -4921,6 +4930,11 @@ ValueList Operands; BasicBlock *IBB = PH->getIncomingBlock(i); + if (!DT->isReachableFromEntry(IBB)) { + NewPhi->addIncoming(PoisonValue::get(NewPhi->getType()), IBB); + continue; + } + if (!VisitedBBs.insert(IBB).second) { NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB); continue; @@ -5621,10 +5635,10 @@ // Sort blocks by domination. This ensures we visit a block after all blocks // dominating it are visited. - llvm::stable_sort(CSEWorkList, - [this](const DomTreeNode *A, const DomTreeNode *B) { - return DT->properlyDominates(A, B); - }); + llvm::sort(CSEWorkList, [](const DomTreeNode *A, const DomTreeNode *B) { + assert((A == B) == (A->getDFSNumIn() == B->getDFSNumIn())); + return A->getDFSNumIn() < B->getDFSNumIn(); + }); // Perform O(N^2) search over the gather sequences and merge identical // instructions. TODO: We can further optimize this scan if we split the @@ -6524,6 +6538,9 @@ // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to // delete instructions. + // Update DFS numbers now so that we can use them for ordering. + DT->updateDFSNumbers(); + // Scan the blocks in the function in post order. for (auto BB : post_order(&F.getEntryBlock())) { collectSeedInstructions(BB); diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordering-bug.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordering-bug.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/ordering-bug.ll @@ -0,0 +1,84 @@ +; RUN: opt -mtriple=i386-linux-gnu -slp-vectorizer -S %s | FileCheck %s + +%struct.a = type { [2 x i64] } + +@a = external global %struct.a +@b = external global %struct.a +@c = external global %struct.a + +define void @f(i1 %x) #0 { +entry: +; CHECK-LABEL: entry: + %a0 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @a, i32 0, i32 0, i32 0), align 8 + %a1 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @a, i32 0, i32 0, i32 1), align 8 + ; CHECK-NEXT: [[a:%.+]] = load <2 x i64>, <2 x i64>* bitcast (%struct.a* @a to <2 x i64>*), align 8 + + br i1 %x, label %while.body.lr.ph, label %while.end + ; CHECK-NEXT: br i1 %x, label %while.body.lr.ph, label %while.end + +while.body.lr.ph: +; CHECK-LABEL: while.body.lr.ph: + ; CHECK-NEXT: [[a1:%.+]] = extractelement <2 x i64> [[a]], i32 1 + + %icmp.a1 = icmp eq i64 %a1, 0 + ; CHECK-NEXT: %icmp.a1 = icmp eq i64 [[a1]], 0 + + %b0 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @b, i32 0, i32 0, i32 0), align 8 + %b1 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @b, i32 0, i32 0, i32 1), align 8 + ; CHECK-NEXT: [[b:%.+]] = load <2 x i64>, <2 x i64>* bitcast (%struct.a* @b to <2 x i64>*), align 8 + + %c0 = select i1 %icmp.a1, i64 %b0, i64 %a0 + %c1 = select i1 %icmp.a1, i64 %b1, i64 %a1 + ; CHECK-NEXT: [[icmp_a1_tmp:%.+]] = insertelement <2 x i1> poison, i1 %icmp.a1, i32 0 + ; CHECK-NEXT: [[icmp_a1:%.+]] = insertelement <2 x i1> [[icmp_a1_tmp]], i1 %icmp.a1, i32 1 + ; CHECK-NEXT: [[c:%.+]] = select <2 x i1> [[icmp_a1]], <2 x i64> [[b]], <2 x i64> [[a]] + + br label %while.end + ; CHECK-NEXT: br label %while.end + +while.end: +; CHECK-LABEL: while.end: + + %d0 = phi i64 [ %a0, %entry ], [ %c0, %while.body.lr.ph ] + %d1 = phi i64 [ %a1, %entry ], [ %c1, %while.body.lr.ph ] + ; CHECK-NEXT: [[d:%.+]] = phi <2 x i64> [ [[a]], %entry ], [ [[c]], %while.body.lr.ph ] + + %e0 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @c, i32 0, i32 0, i32 0), align 8 + %e1 = load i64, i64* getelementptr inbounds (%struct.a, %struct.a* @c, i32 0, i32 0, i32 1), align 8 + ; CHECK-NEXT: [[e:%.+]] = load <2 x i64>, <2 x i64>* bitcast (%struct.a* @c to <2 x i64>*), align 8 + + %icmp.d0 = icmp eq i64 %d0, 0 + ; CHECK-NEXT: [[d0:%.+]] = extractelement <2 x i64> [[d]], i32 0 + ; CHECK-NEXT: %icmp.d0 = icmp eq i64 [[d0]], 0 + + br i1 %icmp.d0, label %if.end, label %if.then + ; CHEKC-NEXT: br i1 %icmp.d0, label %if.end, label %if.then + +if.then: +; CHECK-LABEL: if.then: + + %and0.tmp = and i64 %d0, 8 + ; CHECK-NEXT: %and0.tmp = and i64 [[d0]], 8 + + %and0 = and i64 %and0.tmp, %e0 + %and1 = and i64 %e1, %d1 + ; CHECK-NEXT: [[d1:%.+]] = extractelement <2 x i64> [[d]], i32 1 + ; CHECK-NEXT: [[and_tmp0:%.+]] = insertelement <2 x i64> poison, i64 %and0.tmp, i32 0 + ; CHECK-NEXT: [[and_tmp1:%.+]] = insertelement <2 x i64> [[and_tmp0]], i64 [[d1]], i32 1 + ; CHECK-NEXT: [[and:%.+]] = and <2 x i64> [[and_tmp1]], [[e]] + + store i64 %and0, i64* getelementptr inbounds (%struct.a, %struct.a* @a, i32 0, i32 0, i32 0), align 8 + store i64 %and1, i64* getelementptr inbounds (%struct.a, %struct.a* @a, i32 0, i32 0, i32 1), align 8 + ; CHECK-NEXT: store <2 x i64> [[and]], <2 x i64>* bitcast (%struct.a* @a to <2 x i64>*), align 8 + + br label %if.end + ; CHECK-NEXT: br label %if.end + +if.end: +; CHECK-LABEL: if.end: + + ret void + ; CHECK-NEXT: ret void +} + +attributes #0 = { "target-features"="+sse2" } diff --git a/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll --- a/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/unreachable.ll @@ -23,17 +23,12 @@ ; CHECK-NEXT: [[T10:%.*]] = load i32, i32* [[T9]], align 4 ; CHECK-NEXT: br label [[BB2]] ; CHECK: bb2: -; CHECK-NEXT: [[T1_0:%.*]] = phi i32 [ [[T4]], [[BB1:%.*]] ], [ 2, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[T2_0:%.*]] = phi i32 [ [[T6]], [[BB1]] ], [ 2, [[ENTRY]] ] -; CHECK-NEXT: [[T3_0:%.*]] = phi i32 [ [[T8]], [[BB1]] ], [ 2, [[ENTRY]] ] -; CHECK-NEXT: [[T4_0:%.*]] = phi i32 [ [[T10]], [[BB1]] ], [ 2, [[ENTRY]] ] -; CHECK-NEXT: store i32 [[T1_0]], i32* [[X]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ poison, [[BB1:%.*]] ], [ , [[ENTRY:%.*]] ] ; CHECK-NEXT: [[T12:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1 -; CHECK-NEXT: store i32 [[T2_0]], i32* [[T12]], align 4 ; CHECK-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2 -; CHECK-NEXT: store i32 [[T3_0]], i32* [[T13]], align 4 ; CHECK-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3 -; CHECK-NEXT: store i32 [[T4_0]], i32* [[T14]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[X]] to <4 x i32>* +; CHECK-NEXT: store <4 x i32> [[TMP0]], <4 x i32>* [[TMP1]], align 4 ; CHECK-NEXT: ret void ; entry: