Index: lib/Analysis/LoopAccessAnalysis.cpp =================================================================== --- lib/Analysis/LoopAccessAnalysis.cpp +++ lib/Analysis/LoopAccessAnalysis.cpp @@ -65,6 +65,14 @@ "loop-access analysis (default = 100)"), cl::init(100)); +/// \brief Enable the store-load forwarding performance optimization. This +/// option can be disabled for correctness testing. +static cl::opt EnableStoreLoadForwardOpt( + "enable-store-load-forwarding-opt", cl::Hidden, + cl::desc( + "Enable store-load forwarding optimization in loop-access analysis"), + cl::init(true)); + bool VectorizerParams::isInterleaveForced() { return ::VectorizationInterleave.getNumOccurrences() > 0; } @@ -1204,7 +1212,9 @@ if (Val.isNegative()) { bool IsTrueDataDependence = (AIsWrite && !BIsWrite); if (IsTrueDataDependence && - (couldPreventStoreLoadForward(Val.abs().getZExtValue(), TypeByteSize) || + ((EnableStoreLoadForwardOpt && + couldPreventStoreLoadForward(Val.abs().getZExtValue(), + TypeByteSize)) || ATy != BTy)) { DEBUG(dbgs() << "LAA: Forward but may prevent st->ld forwarding\n"); return Dependence::ForwardButPreventsForwarding; @@ -1309,7 +1319,7 @@ Distance < MaxSafeDepDistBytes ? Distance : MaxSafeDepDistBytes; bool IsTrueDataDependence = (!AIsWrite && BIsWrite); - if (IsTrueDataDependence && + if (IsTrueDataDependence && EnableStoreLoadForwardOpt && couldPreventStoreLoadForward(Distance, TypeByteSize)) return Dependence::BackwardVectorizableButPreventsForwarding; Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -854,6 +854,14 @@ return InterleaveGroupMap.count(Instr); } + /// \brief Return the maximum interleave factor among all interleaved groups. + unsigned getMaxInterleaveFactor() const { + auto MaxFactor = 1u; + for (auto &Entry : InterleaveGroupMap) + MaxFactor = std::max(MaxFactor, Entry.second->getFactor()); + return MaxFactor; + } + /// \brief Get the interleave group that \p Instr belongs to. /// /// \returns nullptr if doesn't have such group. @@ -1334,6 +1342,11 @@ return InterleaveInfo.isInterleaved(Instr); } + /// \brief Return the maximum interleave factor among all interleaved groups. + unsigned getMaxInterleaveFactor() const { + return InterleaveInfo.getMaxInterleaveFactor(); + } + /// \brief Get the interleaved access group that \p Instr belongs to. const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) { return InterleaveInfo.getInterleaveGroup(Instr); @@ -5184,7 +5197,8 @@ unsigned WidestRegister = TTI.getRegisterBitWidth(true); unsigned MaxSafeDepDist = -1U; if (Legal->getMaxSafeDepDistBytes() != -1U) - MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8; + MaxSafeDepDist = + Legal->getMaxSafeDepDistBytes() * 8 / Legal->getMaxInterleaveFactor(); WidestRegister = ((WidestRegister < MaxSafeDepDist) ? WidestRegister : MaxSafeDepDist); unsigned MaxVectorSize = WidestRegister / WidestType; Index: test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll =================================================================== --- /dev/null +++ test/Transforms/LoopVectorize/AArch64/max-vf-for-interleaved.ll @@ -0,0 +1,56 @@ +; RUN: opt < %s -force-vector-interleave=1 -enable-store-load-forwarding-opt=false -loop-vectorize -dce -instcombine -S | FileCheck %s + +target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" +target triple = "aarch64--linux-gnu" + +%struct.pair = type { i32, i32 } + +; Check vectorization of interleaved access groups with positive dependence +; distances. In this test, the maximum safe dependence distance for +; vectorization is 16 bytes. Normally, this would lead to a maximum VF of 4. +; However, for interleaved groups, the effective VF is VF * IF, where IF is the +; interleave factor. Here, the maximum safe dependence distance is recomputed +; as 16 / IF bytes, resulting in VF=2. Since IF=2, we should generate <4 x i32> +; loads and stores instead of <8 x i32> accesses. +; +; Note: LAA's store-load forwarding optimization has to be disabled for this +; test to be vectorized. + +; struct pair { +; short x; +; short y; +; }; +; +; void max_vf(struct pair *restrict p) { +; for (int i = 0; i < 1000; i++) { +; p[i + 2].x = p[i].x +; p[i + 2].y = p[i].y +; } +; } + +; CHECK-LABEL: @max_vf +; CHECK: load <4 x i32> +; CHECK: store <4 x i32> + +define void @max_vf(%struct.pair* noalias nocapture %p) { +entry: + br label %for.body + +for.body: + %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ] + %0 = add nuw nsw i64 %i, 2 + %p_i.x = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %i, i32 0 + %p_i_plus_2.x = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %0, i32 0 + %1 = load i32, i32* %p_i.x, align 4 + store i32 %1, i32* %p_i_plus_2.x, align 4 + %p_i.y = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %i, i32 1 + %p_i_plus_2.y = getelementptr inbounds %struct.pair, %struct.pair* %p, i64 %0, i32 1 + %2 = load i32, i32* %p_i.y, align 4 + store i32 %2, i32* %p_i_plus_2.y, align 4 + %i.next = add nuw nsw i64 %i, 1 + %cond = icmp eq i64 %i.next, 1000 + br i1 %cond, label %for.exit, label %for.body + +for.exit: + ret void +}