diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3872,15 +3872,40 @@ TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx, ReuseShuffleIndicies); LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n"); - TE->setOperandsInOrder(); - for (unsigned i = 0, e = 2; i < e; ++i) { - ValueList Operands; - // Prepare the operand vector. - for (Value *V : VL) - Operands.push_back(cast(V)->getOperand(i)); - - buildTree_rec(Operands, Depth + 1, {TE, i}); + SmallVector Operands(2); + // Prepare the operand vector for pointer operands. + for (Value *V : VL) + Operands.front().push_back( + cast(V)->getPointerOperand()); + TE->setOperand(0, Operands.front()); + // Need to cast all indices to the same type before vectorization to + // avoid crash. + // Required to be able to find correct matches between different gather + // nodes and reuse the vectorized values rather than trying to gather them + // again. + const int IndexIdx = 1; + Type *VL0Ty = VL0->getOperand(IndexIdx)->getType(); + Type *Ty = all_of(VL, + [VL0Ty](Value *V) { + return VL0Ty == cast(V) + ->getOperand(IndexIdx) + ->getType(); + }) + ? VL0Ty + : DL->getIndexType(cast(VL0) + ->getPointerOperandType() + ->getScalarType()); + // Prepare the operand vector. + for (Value *V : VL) { + auto *Op = cast(V)->getOperand(IndexIdx); + auto *CI = cast(Op); + Operands.back().push_back(ConstantExpr::getIntegerCast( + CI, Ty, CI->getValue().isSignBitSet())); } + TE->setOperand(IndexIdx, Operands.back()); + + for (unsigned I = 0, Ops = Operands.size(); I < Ops; ++I) + buildTree_rec(Operands[I], Depth + 1, {TE, I}); return; } case Instruction::Store: { @@ -6283,34 +6308,18 @@ return V; } case Instruction::GetElementPtr: { + auto *GEP0 = cast(VL0); setInsertPointAfterBundle(E); Value *Op0 = vectorizeTree(E->getOperand(0)); - std::vector OpVecs; - for (int j = 1, e = cast(VL0)->getNumOperands(); j < e; - ++j) { - ValueList &VL = E->getOperand(j); - // Need to cast all elements to the same type before vectorization to - // avoid crash. - Type *VL0Ty = VL0->getOperand(j)->getType(); - Type *Ty = llvm::all_of( - VL, [VL0Ty](Value *V) { return VL0Ty == V->getType(); }) - ? VL0Ty - : DL->getIndexType(cast(VL0) - ->getPointerOperandType() - ->getScalarType()); - for (Value *&V : VL) { - auto *CI = cast(V); - V = ConstantExpr::getIntegerCast(CI, Ty, - CI->getValue().isSignBitSet()); - } - Value *OpVec = vectorizeTree(VL); + SmallVector OpVecs; + for (int J = 1, N = GEP0->getNumOperands(); J < N; ++J) { + Value *OpVec = vectorizeTree(E->getOperand(J)); OpVecs.push_back(OpVec); } - Value *V = Builder.CreateGEP( - cast(VL0)->getSourceElementType(), Op0, OpVecs); + Value *V = Builder.CreateGEP(GEP0->getSourceElementType(), Op0, OpVecs); if (Instruction *I = dyn_cast(V)) V = propagateMetadata(I, E->Scalars);