diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -198,7 +198,7 @@ return MaxSafeVectorWidthInBits == UINT_MAX; } - /// The smallest ddependence distance in bytes in the loop. This may not be + /// The smallest dependence distance in bytes in the loop. This may not be /// the same as the maximum number of bytes that are safe to operate on /// simultaneously. uint64_t getMinDepDistBytes() { return MinDepDistBytes; } @@ -276,7 +276,7 @@ /// The program order index to be used for the next instruction. unsigned AccessIdx = 0; - /// The smallest ddependence distance in bytes in the loop. This may not be + /// The smallest dependence distance in bytes in the loop. This may not be /// the same as the maximum number of bytes that are safe to operate on /// simultaneously. uint64_t MinDepDistBytes = 0; @@ -325,8 +325,10 @@ /// forwarding. /// /// \return false if we shouldn't vectorize at all or avoid larger - /// vectorization factors by limiting MinDepDistBytes. - bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize); + /// vectorization factors by limiting MinDepDistBytes and + /// MaxSafeVectorWidthInBits. + bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize, + uint64_t Stride); /// Updates the current safety status with \p S. We can go from Safe to /// either PossiblySafeWithRtChecks or Unsafe and from diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1662,7 +1662,8 @@ } bool MemoryDepChecker::couldPreventStoreLoadForward(uint64_t Distance, - uint64_t TypeByteSize) { + uint64_t TypeByteSize, + uint64_t Stride) { // If loads occur at a distance that is not a multiple of a feasible vector // factor store-load forwarding does not take place. // Positive dependences might cause troubles because vectorizing them might @@ -1700,8 +1701,14 @@ if (MaxVFWithoutSLForwardIssues < MinDepDistBytes && MaxVFWithoutSLForwardIssues != - VectorizerParams::MaxVectorWidth * TypeByteSize) + VectorizerParams::MaxVectorWidth * TypeByteSize) { MinDepDistBytes = MaxVFWithoutSLForwardIssues; + // An update to MaxSafeDepDistBytes requires an update to + // MaxSafeVectorWidthInBits. + uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * Stride); + uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8; + MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits); + } return false; } @@ -1900,7 +1907,8 @@ if (Val.isNegative()) { bool IsTrueDataDependence = (AIsWrite && !BIsWrite); if (IsTrueDataDependence && EnableForwardingConflictDetection && - (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) || + (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize, + Stride) || !HasSameSize)) { LLVM_DEBUG(dbgs() << "LAA: Forward but may prevent st->ld forwarding\n"); return Dependence::ForwardButPreventsForwarding; @@ -1996,16 +2004,19 @@ MinDepDistBytes = std::min(static_cast(Distance), MinDepDistBytes); - bool IsTrueDataDependence = (!AIsWrite && BIsWrite); - if (IsTrueDataDependence && EnableForwardingConflictDetection && - couldPreventStoreLoadForward(Distance, TypeByteSize)) - return Dependence::BackwardVectorizableButPreventsForwarding; - + // An update to MaxSafeDepDistBytes requires an update to + // MaxSafeVectorWidthInBits. uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * Stride); LLVM_DEBUG(dbgs() << "LAA: Positive distance " << Val.getSExtValue() << " with max VF = " << MaxVF << '\n'); uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8; MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits); + + bool IsTrueDataDependence = (!AIsWrite && BIsWrite); + if (IsTrueDataDependence && EnableForwardingConflictDetection && + couldPreventStoreLoadForward(Distance, TypeByteSize, Stride)) + return Dependence::BackwardVectorizableButPreventsForwarding; + return Dependence::BackwardVectorizable; }