Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -5489,6 +5489,20 @@ !0 = !{!"llvm.loop.vectorize.predicate.enable", i1 0} !1 = !{!"llvm.loop.vectorize.predicate.enable", i1 1} +'``llvm.loop.vectorize.ivdep.enable``' Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This metadata indicates to the vectorizer to ignore dependencies between +memory accesses which have not been determined to be safe or unsafe for +vectorization. The first operand is the string +``llvm.loop.vectorize.ivdep.enable`` and the second operand is a bit. +A value of 1 implies that the functionality of this metadata is enabled +for the loop. + +.. code-block:: llvm + + !0 = !{!"llvm.loop.vectorize.ivdep.enable", i1 1} + '``llvm.loop.vectorize.width``' Metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Index: llvm/include/llvm/Analysis/LoopAccessAnalysis.h =================================================================== --- llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -201,7 +201,7 @@ /// /// Only checks sets with elements in \p CheckDeps. bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps, - const ValueToValueMap &Strides); + const ValueToValueMap &Strides, bool ignUnDep); /// No memory dependence was encountered that would inhibit /// vectorization. @@ -519,8 +519,13 @@ AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI); /// Return true we can analyze the memory accesses in the loop and there are - /// no memory dependence cycles. - bool canVectorizeMemory() const { return CanVecMem; } + /// no memory dependence cycles. If Ivdep is enabled,consider Unknown + /// dependence as safe. + bool canVectorizeMemory(bool ignoreUnknown = false) const { + if (ignoreUnknown) + return CanVecMemIvdep; + return CanVecMem; + } /// Return true if there is a convergent operation in the loop. There may /// still be reported runtime pointer checks that would be required, but it is @@ -645,8 +650,10 @@ uint64_t MaxSafeDepDistBytes; - /// Cache the result of analyzeLoop. + /// Cache the result of analyzeLoop. CanVecMemIvdep is used instead of + /// CanVecMem if Ivdep is enabled. bool CanVecMem; + bool CanVecMemIvdep; bool HasConvergentOp; /// Indicator that there are non vectorizable stores to a uniform address. Index: llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h =================================================================== --- llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -44,7 +44,7 @@ /// careful NOT to add them if the user hasn't specifically asked so. class LoopVectorizeHints { enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED, - HK_PREDICATE }; + HK_PREDICATE, HK_IVDEP }; /// Hint - associates name and validation with the hint value. struct Hint { @@ -73,6 +73,9 @@ /// Vector Predicate Hint Predicate; + /// Ignore Vector dependencies + Hint Ivdep; + /// Return the loop metadata prefix. static StringRef Prefix() { return "llvm.loop."; } @@ -102,6 +105,7 @@ unsigned getInterleave() const { return Interleave.Value; } unsigned getIsVectorized() const { return IsVectorized.Value; } unsigned getPredicate() const { return Predicate.Value; } + unsigned getIvdep() const { return Ivdep.Value; } enum ForceKind getForce() const { if ((ForceKind)Force.Value == FK_Undefined && hasDisableAllTransformsHint(TheLoop)) Index: llvm/lib/Analysis/LoopAccessAnalysis.cpp =================================================================== --- llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1633,10 +1633,12 @@ bool MemoryDepChecker::areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps, - const ValueToValueMap &Strides) { + const ValueToValueMap &Strides, + bool ignUnDep) { MaxSafeDepDistBytes = -1; SmallPtrSet Visited; + Status = VectorizationSafetyStatus::Safe; for (MemAccessInfo CurAccess : CheckDeps) { if (Visited.count(CurAccess)) continue; @@ -1678,7 +1680,13 @@ Dependence::DepType Type = isDependent(*A.first, A.second, *B.first, B.second, Strides); - mergeInStatus(Dependence::isSafeForVectorization(Type)); + // Update safety status depending on whether the Dependence type + // is safe. If Unknown Dependence type is to be considered safe, + // do not update safety status. + if (!ignUnDep || + !(Dependence::isSafeForVectorization(Type) == + VectorizationSafetyStatus::PossiblySafeWithRtChecks)) + mergeInStatus(Dependence::isSafeForVectorization(Type)); // Gather dependences unless we accumulated MaxDependences // dependences. In that case return as soon as we find the first @@ -1823,6 +1831,7 @@ // operation, found in this loop, no reason to continue the search. if (HasComplexMemInst && HasConvergentOp) { CanVecMem = false; + CanVecMemIvdep = false; return; } @@ -1896,6 +1905,7 @@ if (HasComplexMemInst) { CanVecMem = false; + CanVecMemIvdep = false; return; } @@ -1907,6 +1917,7 @@ if (!Stores.size()) { LLVM_DEBUG(dbgs() << "LAA: Found a read-only loop!\n"); CanVecMem = true; + CanVecMemIvdep = true; return; } @@ -1953,6 +1964,7 @@ dbgs() << "LAA: A loop annotated parallel, ignore memory dependency " << "checks.\n"); CanVecMem = true; + CanVecMemIvdep = true; return; } @@ -1996,6 +2008,7 @@ if (NumReadWrites == 1 && NumReads == 0) { LLVM_DEBUG(dbgs() << "LAA: Found a write-only loop!\n"); CanVecMem = true; + CanVecMemIvdep = true; return; } @@ -2012,6 +2025,7 @@ LLVM_DEBUG(dbgs() << "LAA: We can't vectorize because we can't find " << "the array bounds.\n"); CanVecMem = false; + CanVecMemIvdep = false; return; } @@ -2019,10 +2033,18 @@ dbgs() << "LAA: May be able to perform a memory runtime check if needed.\n"); CanVecMem = true; + CanVecMemIvdep = true; if (Accesses.isDependencyCheckNeeded()) { LLVM_DEBUG(dbgs() << "LAA: Checking memory dependencies\n"); + + // Check safety of dependences if Unknown dependence is considered safe + CanVecMemIvdep = DepChecker->areDepsSafe( + DependentAccesses, Accesses.getDependenciesToCheck(), SymbolicStrides, + true); + CanVecMem = DepChecker->areDepsSafe( - DependentAccesses, Accesses.getDependenciesToCheck(), SymbolicStrides); + DependentAccesses, Accesses.getDependenciesToCheck(), SymbolicStrides, + false); MaxSafeDepDistBytes = DepChecker->getMaxSafeDepDistBytes(); if (!CanVecMem && DepChecker->shouldRetryWithRuntimeCheck()) { @@ -2057,10 +2079,11 @@ LLVM_DEBUG(dbgs() << "LAA: We can't vectorize because a runtime check " "would be needed with a convergent operation\n"); CanVecMem = false; + CanVecMemIvdep = false; return; } - if (CanVecMem) + if (CanVecMem || CanVecMemIvdep) LLVM_DEBUG( dbgs() << "LAA: No unsafe dependent memory operations in loop. We" << (PtrRtChecking->Need ? "" : " don't") @@ -2348,14 +2371,14 @@ PtrRtChecking(std::make_unique(SE)), DepChecker(std::make_unique(*PSE, L)), TheLoop(L), NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1), CanVecMem(false), - HasConvergentOp(false), + CanVecMemIvdep(false), HasConvergentOp(false), HasDependenceInvolvingLoopInvariantAddress(false) { if (canAnalyzeLoop()) analyzeLoop(AA, LI, TLI, DT); } void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const { - if (CanVecMem) { + if (CanVecMem || CanVecMemIvdep) { OS.indent(Depth) << "Memory dependences are safe"; if (MaxSafeDepDistBytes != -1ULL) OS << " with a maximum dependence distance of " << MaxSafeDepDistBytes Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -61,6 +61,8 @@ case HK_ISVECTORIZED: case HK_PREDICATE: return (Val == 0 || Val == 1); + case HK_IVDEP: + return (Val == 1); } return false; } @@ -72,7 +74,8 @@ Interleave("interleave.count", InterleaveOnlyWhenForced, HK_UNROLL), Force("vectorize.enable", FK_Undefined, HK_FORCE), IsVectorized("isvectorized", 0, HK_ISVECTORIZED), - Predicate("vectorize.predicate.enable", 0, HK_PREDICATE), TheLoop(L), + Predicate("vectorize.predicate.enable", 0, HK_PREDICATE), + Ivdep("vectorize.ivdep.enable", 0, HK_IVDEP), TheLoop(L), ORE(ORE) { // Populate values with existing loop metadata. getHintsFromMetadata(); @@ -224,7 +227,8 @@ return; unsigned Val = C->getZExtValue(); - Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized, &Predicate}; + Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized, &Predicate, + &Ivdep}; for (auto H : Hints) { if (Name == H->Name) { if (H->validate(Val)) @@ -833,7 +837,7 @@ "loop not vectorized: ", *LAR); }); } - if (!LAI->canVectorizeMemory()) + if (!LAI->canVectorizeMemory(Hints->getIvdep())) return false; if (LAI->hasDependenceInvolvingLoopInvariantAddress()) {