Index: llvm/include/llvm/Analysis/IVDescriptors.h =================================================================== --- llvm/include/llvm/Analysis/IVDescriptors.h +++ llvm/include/llvm/Analysis/IVDescriptors.h @@ -329,6 +329,11 @@ : Instruction::BinaryOpsEnd; } + Type *getElementType() const { + assert(IK == IK_PtrInduction && "Only pointer induction has element type"); + return ElementType; + } + /// Returns a reference to the type cast instructions in the induction /// update chain, that are redundant when guarded with a runtime /// SCEV overflow check. @@ -340,6 +345,7 @@ /// Private constructor - used by \c isInductionPHI. InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, BinaryOperator *InductionBinOp = nullptr, + Type *ElementType = nullptr, SmallVectorImpl *Casts = nullptr); /// Start value. @@ -350,6 +356,8 @@ const SCEV *Step = nullptr; // Instruction that advances induction variable. BinaryOperator *InductionBinOp = nullptr; + // Element type for pointer induction variables. + Type *ElementType = nullptr; // Instructions used for type-casts of the induction variable, // that are redundant when guarded with a runtime SCEV overflow check. SmallVector RedundantCasts; Index: llvm/lib/Analysis/IVDescriptors.cpp =================================================================== --- llvm/lib/Analysis/IVDescriptors.cpp +++ llvm/lib/Analysis/IVDescriptors.cpp @@ -963,8 +963,10 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step, BinaryOperator *BOp, + Type *ElementType, SmallVectorImpl *Casts) - : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) { + : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp), + ElementType(ElementType) { assert(IK != IK_NoInduction && "Not an induction"); // Start value type should match the induction kind and the value @@ -992,6 +994,11 @@ InductionBinOp->getOpcode() == Instruction::FSub))) && "Binary opcode should be specified for FP induction"); + if (IK == IK_PtrInduction) + assert(ElementType && "Pointer induction must have element type"); + else + assert(!ElementType && "Non-pointer induction cannot have element type"); + if (Casts) { for (auto &Inst : *Casts) { RedundantCasts.push_back(Inst); @@ -1239,8 +1246,6 @@ BasicBlock *Latch = AR->getLoop()->getLoopLatch(); if (!Latch) return false; - BinaryOperator *BOp = - dyn_cast(Phi->getIncomingValueForBlock(Latch)); const SCEV *Step = AR->getStepRecurrence(*SE); // Calculate the pointer stride and check if it is consecutive. @@ -1250,8 +1255,10 @@ return false; if (PhiTy->isIntegerTy()) { + BinaryOperator *BOp = + dyn_cast(Phi->getIncomingValueForBlock(Latch)); D = InductionDescriptor(StartValue, IK_IntInduction, Step, BOp, - CastsToIgnore); + /* ElementType */ nullptr, CastsToIgnore); return true; } @@ -1260,15 +1267,16 @@ if (!ConstStep) return false; - ConstantInt *CV = ConstStep->getValue(); - Type *PointerElementType = PhiTy->getPointerElementType(); - // The pointer stride cannot be determined if the pointer element type is not - // sized. - if (!PointerElementType->isSized()) + // Always use i8 element type for opaque pointer inductions. + PointerType *PtrTy = cast(PhiTy); + Type *ElementType = PtrTy->isOpaque() ? Type::getInt8Ty(PtrTy->getContext()) + : PtrTy->getElementType(); + if (!ElementType->isSized()) return false; + ConstantInt *CV = ConstStep->getValue(); const DataLayout &DL = Phi->getModule()->getDataLayout(); - int64_t Size = static_cast(DL.getTypeAllocSize(PointerElementType)); + int64_t Size = static_cast(DL.getTypeAllocSize(ElementType)); if (!Size) return false; @@ -1277,6 +1285,7 @@ return false; auto *StepValue = SE->getConstant(CV->getType(), CVSize / Size, true /* signed */); - D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, BOp); + D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, + /* BinOp */ nullptr, ElementType); return true; } Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3448,7 +3448,7 @@ assert(isa(Step) && "Expected constant step for pointer induction"); return B.CreateGEP( - StartValue->getType()->getPointerElementType(), StartValue, + ID.getElementType(), StartValue, CreateMul(Index, Exp.expandCodeFor(Step, Index->getType()->getScalarType(), GetInsertPoint()))); @@ -4811,7 +4811,7 @@ Value *NumUnrolledElems = Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF)); Value *InductionGEP = GetElementPtrInst::Create( - ScStValueType->getPointerElementType(), NewPointerPhi, + II.getElementType(), NewPointerPhi, Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind", InductionLoc); NewPointerPhi->addIncoming(InductionGEP, LoopLatch); @@ -4830,7 +4830,7 @@ Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType)); Value *GEP = Builder.CreateGEP( - ScStValueType->getPointerElementType(), NewPointerPhi, + II.getElementType(), NewPointerPhi, Builder.CreateMul( StartOffset, Builder.CreateVectorSplat(State.VF, ScalarStepValue), "vector.gep")); Index: llvm/test/Transforms/LoopVectorize/opaque-ptr.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopVectorize/opaque-ptr.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -loop-vectorize -force-vector-width=2 < %s | FileCheck %s + +; TODO: This still crashes with inbounds on the GEPs. +define void @test(ptr %p1.start, ptr %p2.start, ptr %p1.end) { +; CHECK-LABEL: @test( +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[P1:%.*]] = phi ptr [ [[P1_START:%.*]], [[ENTRY:%.*]] ], [ [[P1_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[P2:%.*]] = phi ptr [ [[P2_START:%.*]], [[ENTRY]] ], [ [[P2_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[P1_VAL:%.*]] = load float, ptr [[P1]], align 4 +; CHECK-NEXT: [[P2_VAL:%.*]] = load float, ptr [[P2]], align 4 +; CHECK-NEXT: [[SUM:%.*]] = fadd float [[P1_VAL]], [[P2_VAL]] +; CHECK-NEXT: store float [[SUM]], ptr [[P1]], align 4 +; CHECK-NEXT: [[P1_NEXT]] = getelementptr float, ptr [[P1]], i64 1 +; CHECK-NEXT: [[P2_NEXT]] = getelementptr float, ptr [[P2]], i64 1 +; CHECK-NEXT: [[C:%.*]] = icmp ne ptr [[P1_NEXT]], [[P1_END:%.*]] +; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %p1 = phi ptr [ %p1.start, %entry ], [ %p1.next, %loop ] + %p2 = phi ptr [ %p2.start, %entry ], [ %p2.next, %loop ] + %p1.val = load float, ptr %p1 + %p2.val = load float, ptr %p2 + %sum = fadd float %p1.val, %p2.val + store float %sum, ptr %p1 + %p1.next = getelementptr float, ptr %p1, i64 1 + %p2.next = getelementptr float, ptr %p2, i64 1 + %c = icmp ne ptr %p1.next, %p1.end + br i1 %c, label %loop, label %exit + +exit: + ret void +}