Index: llvm/include/llvm/Analysis/IVDescriptors.h =================================================================== --- llvm/include/llvm/Analysis/IVDescriptors.h +++ llvm/include/llvm/Analysis/IVDescriptors.h @@ -329,6 +329,11 @@ : Instruction::BinaryOpsEnd; } + Type *getElementType() const { + assert(IK == IK_PtrInduction && "Only pointer induction has element type"); + return ElementType; + } + /// Returns a reference to the type cast instructions in the induction /// update chain, that are redundant when guarded with a runtime /// SCEV overflow check. @@ -340,6 +345,7 @@ /// Private constructor - used by \c isInductionPHI. InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, BinaryOperator *InductionBinOp = nullptr, + Type *ElementType = nullptr, SmallVectorImpl *Casts = nullptr); /// Start value. @@ -350,6 +356,8 @@ const SCEV *Step = nullptr; // Instruction that advances induction variable. BinaryOperator *InductionBinOp = nullptr; + // Element type for pointer induction variables. + Type *ElementType = nullptr; // Instructions used for type-casts of the induction variable, // that are redundant when guarded with a runtime SCEV overflow check. SmallVector RedundantCasts; Index: llvm/lib/Analysis/IVDescriptors.cpp =================================================================== --- llvm/lib/Analysis/IVDescriptors.cpp +++ llvm/lib/Analysis/IVDescriptors.cpp @@ -964,8 +964,10 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, BinaryOperator *BOp, + Type *ElementType, SmallVectorImpl *Casts) - : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) { + : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp), + ElementType(ElementType) { assert(IK != IK_NoInduction && "Not an induction"); // Start value type should match the induction kind and the value @@ -993,6 +995,11 @@ InductionBinOp->getOpcode() == Instruction::FSub))) && "Binary opcode should be specified for FP induction"); + if (IK == IK_PtrInduction) + assert(ElementType && "Pointer induction must have element type"); + else + assert(!ElementType && "Non-pointer induction cannot have element type"); + if (Casts) { for (auto &Inst : *Casts) { RedundantCasts.push_back(Inst); @@ -1240,8 +1247,6 @@ BasicBlock *Latch = AR->getLoop()->getLoopLatch(); if (!Latch) return false; - BinaryOperator *BOp = - dyn_cast(Phi->getIncomingValueForBlock(Latch)); const SCEV *Step = AR->getStepRecurrence(*SE); // Calculate the pointer stride and check if it is consecutive. @@ -1251,8 +1256,10 @@ return false; if (PhiTy->isIntegerTy()) { + BinaryOperator *BOp = + dyn_cast(Phi->getIncomingValueForBlock(Latch)); D = InductionDescriptor(StartValue, IK_IntInduction, Step, BOp, - CastsToIgnore); + /* ElementType */ nullptr, CastsToIgnore); return true; } @@ -1261,15 +1268,18 @@ if (!ConstStep) return false; - ConstantInt *CV = ConstStep->getValue(); - Type *PointerElementType = PhiTy->getPointerElementType(); - // The pointer stride cannot be determined if the pointer element type is not - // sized. - if (!PointerElementType->isSized()) + // We need a GEP to determine the element type. + GEPOperator *GEP = + dyn_cast(Phi->getIncomingValueForBlock(Latch)); + if (!GEP) return false; + ConstantInt *CV = ConstStep->getValue(); + Type *ElementType = GEP->getSourceElementType(); + assert(ElementType->isSized() && "GEP type must be sized"); + const DataLayout &DL = Phi->getModule()->getDataLayout(); - int64_t Size = static_cast(DL.getTypeAllocSize(PointerElementType)); + int64_t Size = static_cast(DL.getTypeAllocSize(ElementType)); if (!Size) return false; @@ -1278,6 +1288,7 @@ return false; auto *StepValue = SE->getConstant(CV->getType(), CVSize / Size, true /* signed */); - D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, BOp); + D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, + /* BinOp */ nullptr, ElementType); return true; } Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3417,7 +3417,7 @@ assert(isa(Step) && "Expected constant step for pointer induction"); return B.CreateGEP( - StartValue->getType()->getPointerElementType(), StartValue, + ID.getElementType(), StartValue, CreateMul(Index, Exp.expandCodeFor(Step, Index->getType()->getScalarType(), GetInsertPoint()))); @@ -4914,7 +4914,7 @@ Value *NumUnrolledElems = Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF)); Value *InductionGEP = GetElementPtrInst::Create( - ScStValueType->getPointerElementType(), NewPointerPhi, + II.getElementType(), NewPointerPhi, Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind", InductionLoc); NewPointerPhi->addIncoming(InductionGEP, LoopLatch); @@ -4933,7 +4933,7 @@ Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType)); Value *GEP = Builder.CreateGEP( - ScStValueType->getPointerElementType(), NewPointerPhi, + II.getElementType(), NewPointerPhi, Builder.CreateMul( StartOffset, Builder.CreateVectorSplat(State.VF, ScalarStepValue), "vector.gep")); Index: llvm/test/Transforms/LoopVectorize/iv_outside_user.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/iv_outside_user.ll +++ llvm/test/Transforms/LoopVectorize/iv_outside_user.ll @@ -82,14 +82,10 @@ ret i32* %ptr.phi } +; This does not get vectorized, because %inc.lag2 doesn't have a straightforward +; GEP step instruction and the element type cannot be determined. ; CHECK-LABEL: @both -; CHECK-LABEL: middle.block: -; CHECK: %[[END:.*]] = sub i64 %n.vec, 1 -; CHECK: %ind.escape = getelementptr i32, i32* %base, i64 %[[END]] -; CHECK-LABEL: for.end: -; CHECK: %[[RET:.*]] = phi i32* [ %inc.lag1, %for.body ], [ %ind.escape, %middle.block ] -; CHECK: ret i32* %[[RET]] - +; CHECK-NOT: vector.body define i32* @both(i32 %k) { entry: %base = getelementptr inbounds i32, i32* undef, i64 1 Index: llvm/test/Transforms/LoopVectorize/opaque-ptr.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/opaque-ptr.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -loop-vectorize -force-vector-width=2 < %s | FileCheck %s + +; TODO: This still crashes with inbounds on the GEPs. +define void @test(ptr %p1.start, ptr %p2.start, ptr %p1.end) { +; CHECK-LABEL: @test( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[P1:%.*]] = phi ptr [ [[P1_START:%.*]], [[ENTRY:%.*]] ], [ [[P1_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[P2:%.*]] = phi ptr [ [[P2_START:%.*]], [[ENTRY]] ], [ [[P2_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[P1_VAL:%.*]] = load float, ptr [[P1]], align 4 +; CHECK-NEXT: [[P2_VAL:%.*]] = load float, ptr [[P2]], align 4 +; CHECK-NEXT: [[SUM:%.*]] = fadd float [[P1_VAL]], [[P2_VAL]] +; CHECK-NEXT: store float [[SUM]], ptr [[P1]], align 4 +; CHECK-NEXT: [[P1_NEXT]] = getelementptr float, ptr [[P1]], i64 1 +; CHECK-NEXT: [[P2_NEXT]] = getelementptr float, ptr [[P2]], i64 1 +; CHECK-NEXT: [[C:%.*]] = icmp ne ptr [[P1_NEXT]], [[P1_END:%.*]] +; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %p1 = phi ptr [ %p1.start, %entry ], [ %p1.next, %loop ] + %p2 = phi ptr [ %p2.start, %entry ], [ %p2.next, %loop ] + %p1.val = load float, ptr %p1 + %p2.val = load float, ptr %p2 + %sum = fadd float %p1.val, %p2.val + store float %sum, ptr %p1 + %p1.next = getelementptr float, ptr %p1, i64 1 + %p2.next = getelementptr float, ptr %p2, i64 1 + %c = icmp ne ptr %p1.next, %p1.end + br i1 %c, label %loop, label %exit + +exit: + ret void +}