Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -623,12 +623,37 @@ /// RuntimePointerCheck class. class LoopAccessAnalysis { public: + /// \brief Collection of parameters used from the vectorizer. + struct VectorizerParams { + /// \brief Maximum simd width. + unsigned MaxVectorWidth; + + /// \brief VF as overridden by the user. + unsigned VectorizationFactor; + /// \brief Interleave factor as overridden by the user. + unsigned VectorizationInterleave; + + /// \\brief When performing memory disambiguation checks at runtime do not + /// make more than this number of comparisons. + unsigned RuntimeMemoryCheckThreshold; + + VectorizerParams(unsigned MaxVectorWidth, + unsigned VectorizationFactor, + unsigned VectorizationInterleave, + unsigned RuntimeMemoryCheckThreshold) : + MaxVectorWidth(MaxVectorWidth), + VectorizationFactor(VectorizationFactor), + VectorizationInterleave(VectorizationInterleave), + RuntimeMemoryCheckThreshold(RuntimeMemoryCheckThreshold) {} + }; + LoopAccessAnalysis(Function *F, Loop *L, ScalarEvolution *SE, const DataLayout *DL, const TargetLibraryInfo *TLI, - AliasAnalysis *AA, DominatorTree *DT) : + AliasAnalysis *AA, DominatorTree *DT, + const VectorizerParams &VectParams) : TheFunction(F), TheLoop(L), SE(SE), DL(DL), TLI(TLI), AA(AA), DT(DT), - NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1U) { - } + NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1U), + VectParams(VectParams) {} /// Return true we can analyze the memory accesses in the loop and there are /// no memory dependence cycles. Replaces symbolic strides using Strides. @@ -665,6 +690,9 @@ unsigned NumStores; unsigned MaxSafeDepDistBytes; + + /// \brief Vectorizer parameters used by the analysis. + VectorizerParams VectParams; }; } // end anonymous namespace @@ -689,7 +717,10 @@ const TargetTransformInfo *TTI) : NumPredStores(0), TheLoop(L), SE(SE), DL(DL), TLI(TLI), TheFunction(F), TTI(TTI), Induction(nullptr), WidestIndTy(nullptr), - LAA(F, L, SE, DL, TLI, AA, DT), HasFunNoNaNAttr(false) { + LAA(F, L, SE, DL, TLI, AA, DT, + {MaxVectorWidth, VectorizationFactor, VectorizationInterleave, + RuntimeMemoryCheckThreshold}), + HasFunNoNaNAttr(false) { } /// This enum represents the kinds of reductions that we support. @@ -4447,9 +4478,10 @@ typedef PointerIntPair MemAccessInfo; typedef SmallPtrSet MemAccessInfoSet; - MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L) + MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L, + const LoopAccessAnalysis::VectorizerParams &VectParams) : SE(Se), DL(Dl), InnermostLoop(L), AccessIdx(0), - ShouldRetryWithRuntimeCheck(false) {} + ShouldRetryWithRuntimeCheck(false), VectParams(VectParams) {} /// \brief Register the location (instructions are given increasing numbers) /// of a write access. @@ -4504,6 +4536,9 @@ /// vectorize this loop with runtime checks. bool ShouldRetryWithRuntimeCheck; + /// \brief Vectorizer parameters used by the analysis. + LoopAccessAnalysis::VectorizerParams VectParams; + /// \brief Check whether there is a plausible dependence between the two /// accesses. /// @@ -4627,7 +4662,7 @@ // Store-load forwarding distance. const unsigned NumCyclesForStoreLoadThroughMemory = 8*TypeByteSize; // Maximum vector factor. - unsigned MaxVFWithoutSLForwardIssues = MaxVectorWidth*TypeByteSize; + unsigned MaxVFWithoutSLForwardIssues = VectParams.MaxVectorWidth*TypeByteSize; if(MaxSafeDepDistBytes < MaxVFWithoutSLForwardIssues) MaxVFWithoutSLForwardIssues = MaxSafeDepDistBytes; @@ -4646,7 +4681,7 @@ } if (MaxVFWithoutSLForwardIssues < MaxSafeDepDistBytes && - MaxVFWithoutSLForwardIssues != MaxVectorWidth*TypeByteSize) + MaxVFWithoutSLForwardIssues != VectParams.MaxVectorWidth*TypeByteSize) MaxSafeDepDistBytes = MaxVFWithoutSLForwardIssues; return false; } @@ -4751,8 +4786,10 @@ unsigned Distance = (unsigned) Val.getZExtValue(); // Bail out early if passed-in parameters make vectorization not feasible. - unsigned ForcedFactor = VectorizationFactor ? VectorizationFactor : 1; - unsigned ForcedUnroll = VectorizationInterleave ? VectorizationInterleave : 1; + unsigned ForcedFactor = (VectParams.VectorizationFactor ? + VectParams.VectorizationFactor : 1); + unsigned ForcedUnroll = (VectParams.VectorizationInterleave ? + VectParams.VectorizationInterleave : 1); // The distance must be bigger than the size needed for a vectorized version // of the operation and the size of the vectorized operation must not be @@ -4835,7 +4872,7 @@ PtrRtCheck.Need = false; const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel(); - MemoryDepChecker DepChecker(SE, DL, TheLoop); + MemoryDepChecker DepChecker(SE, DL, TheLoop, VectParams); // For each block. for (Loop::block_iterator bb = TheLoop->block_begin(), @@ -5003,7 +5040,7 @@ // Check that we did not collect too many pointers or found an unsizeable // pointer. - if (!CanDoRT || NumComparisons > RuntimeMemoryCheckThreshold) { + if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) { PtrRtCheck.reset(); CanDoRT = false; } @@ -5043,14 +5080,14 @@ TheLoop, Strides, true); // Check that we did not collect too many pointers or found an unsizeable // pointer. - if (!CanDoRT || NumComparisons > RuntimeMemoryCheckThreshold) { + if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) { if (!CanDoRT && NumComparisons > 0) emitAnalysis(Report() << "cannot check memory dependencies at runtime"); else emitAnalysis(Report() << NumComparisons << " exceeds limit of " - << RuntimeMemoryCheckThreshold + << VectParams.RuntimeMemoryCheckThreshold << " dependent memory operations checked at runtime"); DEBUG(dbgs() << "LV: Can't vectorize with memory checks\n"); PtrRtCheck.reset();