diff --git a/llvm/docs/AliasAnalysis.rst b/llvm/docs/AliasAnalysis.rst --- a/llvm/docs/AliasAnalysis.rst +++ b/llvm/docs/AliasAnalysis.rst @@ -166,9 +166,25 @@ The ``pointsToConstantMemory`` method returns true if and only if the analysis can prove that the pointer only points to unchanging memory locations -(functions, constant global variables, and the null pointer). This information -can be used to refine mod/ref information: it is impossible for an unchanging -memory location to be modified. +(functions, constant global variables, and the null pointer). The +``getModRefInfoMask`` function uses this to refine mod/ref information: it is +impossible for an unchanging memory location to be modified. + +The ``getModRefInfoMask`` method +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``getModRefInfoMask`` method returns a bound on Mod/Ref information for +the supplied pointer, based on knowledge about whether the pointer points to +globally-constant memory (for which it returns ``NoModRef``) or +locally-invariant memory (for which it returns ``Ref``). Locally-invariant +memory is memory that we know is invariant for the lifetime of its SSA value, +but not necessarily for the life of the program: for example, the memory +pointed to by ``readonly`` ``noalias`` parameters is known-invariant for the +duration of the corresponding function call. Given Mod/Ref information +``MRI`` for a memory location ``Loc``, ``MRI`` can be refined with a +statement like ``MRI &= AA.getModRefInfoMask(Loc);``. Another useful idiom is +``isModSet(AA.getModRefInfoMask(Loc))``; this checks to see if the given +location can be modified at all. .. _never access memory or only read memory: diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -362,7 +362,9 @@ /// Checks whether the given location points to constant memory, or if /// \p OrLocal is true whether it points to a local alloca. - bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false); + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) { + return isNoModRef(getModRefInfoMask(Loc, OrLocal)); + } /// A convenience wrapper around the primary \c pointsToConstantMemory /// interface. @@ -375,6 +377,22 @@ /// \name Simple mod/ref information /// @{ + /// Returns a bitmask that should be unconditionally applied to the ModRef + /// info of a memory location. This allows us to eliminate Mod and/or Ref + /// from the ModRef info based on the knowledge that the memory location + /// points to constant and/or locally-invariant memory. + /// + /// If IgnoreLocals is true, then this method returns NoModRef for memory + /// that points to a local alloca. + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, + bool IgnoreLocals = false); + + /// A convenience wrapper around the primary \c getModRefInfoMask + /// interface. + ModRefInfo getModRefInfoMask(const Value *P, bool IgnoreLocals = false) { + return getModRefInfoMask(MemoryLocation::getBeforeOrAfter(P), IgnoreLocals); + } + /// Get the ModRef info associated with a pointer argument of a call. The /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note /// that these bits do not necessarily account for the overall behavior of @@ -617,6 +635,8 @@ AAQueryInfo &AAQI); bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, bool OrLocal = false); + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals = false); ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2, AAQueryInfo &AAQIP); ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, @@ -684,6 +704,10 @@ bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) { return AA.pointsToConstantMemory(Loc, AAQI, OrLocal); } + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, + bool IgnoreLocals = false) { + return AA.getModRefInfoMask(Loc, AAQI, IgnoreLocals); + } ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { return AA.getModRefInfo(Call, Loc, AAQI); } @@ -756,6 +780,14 @@ /// \name Simple mod/ref information /// @{ + /// Returns a bitmask that should be unconditionally applied to the ModRef + /// info of a memory location. This allows us to eliminate Mod and/or Ref from + /// the ModRef info based on the knowledge that the memory location points to + /// constant and/or locally-invariant memory. + virtual ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) = 0; + /// Get the ModRef info associated with a pointer argument of a callsite. The /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note /// that these bits do not necessarily account for the overall behavior of @@ -809,6 +841,11 @@ return Result.pointsToConstantMemory(Loc, AAQI, OrLocal); } + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals) override { + return Result.getModRefInfoMask(Loc, AAQI, IgnoreLocals); + } + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override { return Result.getArgModRefInfo(Call, ArgIdx); } @@ -864,6 +901,11 @@ return false; } + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals) { + return ModRefInfo::ModRef; + } + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { return ModRefInfo::ModRef; } diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h --- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h @@ -79,6 +79,16 @@ bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, bool OrLocal); + /// Returns a bitmask that should be unconditionally applied to the ModRef + /// info of a memory location. This allows us to eliminate Mod and/or Ref + /// from the ModRef info based on the knowledge that the memory location + /// points to constant and/or locally-invariant memory. + /// + /// If IgnoreLocals is true, then this method returns NoModRef for memory + /// that points to a local alloca. + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals = false); + /// Get the location associated with a pointer argument of a callsite. ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx); diff --git a/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h --- a/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h +++ b/llvm/include/llvm/Analysis/TypeBasedAliasAnalysis.h @@ -40,8 +40,8 @@ AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, AAQueryInfo &AAQI); - bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, - bool OrLocal); + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals); MemoryEffects getMemoryEffects(const CallBase *Call, AAQueryInfo &AAQI); MemoryEffects getMemoryEffects(const Function *F); ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp --- a/llvm/lib/Analysis/AliasAnalysis.cpp +++ b/llvm/lib/Analysis/AliasAnalysis.cpp @@ -147,12 +147,6 @@ return Result; } -bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc, - bool OrLocal) { - SimpleAAQueryInfo AAQIP(*this); - return pointsToConstantMemory(Loc, AAQIP, OrLocal); -} - bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, bool OrLocal) { for (const auto &AA : AAs) @@ -162,6 +156,27 @@ return false; } +ModRefInfo AAResults::getModRefInfoMask(const MemoryLocation &Loc, + bool IgnoreLocals) { + SimpleAAQueryInfo AAQIP(*this); + return getModRefInfoMask(Loc, AAQIP, IgnoreLocals); +} + +ModRefInfo AAResults::getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, bool IgnoreLocals) { + ModRefInfo Result = ModRefInfo::ModRef; + + for (const auto &AA : AAs) { + Result &= AA->getModRefInfoMask(Loc, AAQI, IgnoreLocals); + + // Early-exit the moment we reach the bottom of the lattice. + if (isNoModRef(Result)) + return ModRefInfo::NoModRef; + } + + return Result; +} + ModRefInfo AAResults::getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { ModRefInfo Result = ModRefInfo::ModRef; @@ -253,10 +268,11 @@ Result &= ArgMR | OtherMR; - // If Loc is a constant memory location, the call definitely could not + // Apply the ModRef mask. This ensures that if Loc is a constant memory + // location, we take into account the fact that the call definitely could not // modify the memory location. - if (isModSet(Result) && pointsToConstantMemory(Loc, AAQI, /*OrLocal*/ false)) - Result &= ModRefInfo::Ref; + if (!isNoModRef(Result)) + Result &= getModRefInfoMask(Loc); return Result; } @@ -510,9 +526,11 @@ if (AR == AliasResult::NoAlias) return ModRefInfo::NoModRef; - // If the pointer is a pointer to constant memory, then it could not have - // been modified by this store. - if (pointsToConstantMemory(Loc, AAQI)) + // Examine the ModRef mask. If Mod isn't present, then return NoModRef. + // This ensures that if Loc is a constant memory location, we take into + // account the fact that the store definitely could not modify the memory + // location. + if (!isModSet(getModRefInfoMask(Loc))) return ModRefInfo::NoModRef; } @@ -529,11 +547,10 @@ ModRefInfo AAResults::getModRefInfo(const FenceInst *S, const MemoryLocation &Loc, AAQueryInfo &AAQI) { - // If we know that the location is a constant memory location, the fence - // cannot modify this location. - if (Loc.Ptr && pointsToConstantMemory(Loc, AAQI)) - return ModRefInfo::Ref; - return ModRefInfo::ModRef; + // All we know about a fence instruction is what we get from the ModRef + // mask: if Loc is a constant memory location, the fence definitely could + // not modify it. + return getModRefInfoMask(Loc); } ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, @@ -552,9 +569,9 @@ if (AR == AliasResult::NoAlias) return ModRefInfo::NoModRef; - // If the pointer is a pointer to constant memory, then it could not have + // If the pointer is a pointer to invariant memory, then it could not have // been modified by this va_arg. - if (pointsToConstantMemory(Loc, AAQI)) + if (!isModSet(getModRefInfoMask(Loc, AAQI))) return ModRefInfo::NoModRef; } @@ -572,9 +589,9 @@ const MemoryLocation &Loc, AAQueryInfo &AAQI) { if (Loc.Ptr) { - // If the pointer is a pointer to constant memory, + // If the pointer is a pointer to invariant memory, // then it could not have been modified by this catchpad. - if (pointsToConstantMemory(Loc, AAQI)) + if (!isModSet(getModRefInfoMask(Loc, AAQI))) return ModRefInfo::NoModRef; } @@ -592,9 +609,9 @@ const MemoryLocation &Loc, AAQueryInfo &AAQI) { if (Loc.Ptr) { - // If the pointer is a pointer to constant memory, + // If the pointer is a pointer to invariant memory, // then it could not have been modified by this catchpad. - if (pointsToConstantMemory(Loc, AAQI)) + if (!isModSet(getModRefInfoMask(Loc, AAQI))) return ModRefInfo::NoModRef; } diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp --- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp +++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp @@ -681,9 +681,10 @@ return Decomposed; } -/// Returns whether the given pointer value points to memory that is local to -/// the function, with global constants being considered local to all -/// functions. +/// Returns a bitmask that should be unconditionally applied to the ModRef +/// info of a memory location. This allows us to eliminate Mod and/or Ref +/// from the ModRef info based on the knowledge that the memory location +/// points to constant and/or locally-invariant memory. bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, bool OrLocal) { assert(Visited.empty() && "Visited must be cleared after use!"); @@ -820,6 +821,80 @@ return false; } +ModRefInfo BasicAAResult::getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) { + if (!Loc.Ptr) + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + + assert(Visited.empty() && "Visited must be cleared after use!"); + auto _ = make_scope_exit([&] { Visited.clear(); }); + + unsigned MaxLookup = 8; + SmallVector Worklist; + Worklist.push_back(Loc.Ptr); + ModRefInfo Result = ModRefInfo::NoModRef; + + do { + const Value *V = getUnderlyingObject(Worklist.pop_back_val()); + if (!Visited.insert(V).second) + continue; + + // Ignore allocas if we were instructed to do so. + if (IgnoreLocals && isa(V)) + continue; + + // If the location points to memory that is known to be invariant for + // the life of the underlying SSA value, then we can exclude Mod from + // the set of valid memory effects. + // + // An argument that is marked readonly and noalias is known to be + // invariant while that function is executing. + if (const Argument *Arg = dyn_cast(V)) { + if (Arg->hasNoAliasAttr() && Arg->hasAttribute(Attribute::ReadOnly)) { + Result = Result | ModRefInfo::Ref; + continue; + } + } + + // A global constant can't be mutated. + if (const GlobalVariable *GV = dyn_cast(V)) { + // Note: this doesn't require GV to be "ODR" because it isn't legal for a + // global to be marked constant in some modules and non-constant in + // others. GV may even be a declaration, not a definition. + if (!GV->isConstant()) + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + continue; + } + + // If both select values point to local memory, then so does the select. + if (const SelectInst *SI = dyn_cast(V)) { + Worklist.push_back(SI->getTrueValue()); + Worklist.push_back(SI->getFalseValue()); + continue; + } + + // If all values incoming to a phi node point to local memory, then so does + // the phi. + if (const PHINode *PN = dyn_cast(V)) { + // Don't bother inspecting phi nodes with many operands. + if (PN->getNumIncomingValues() > MaxLookup) + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + append_range(Worklist, PN->incoming_values()); + continue; + } + + // Otherwise be conservative. + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + } while (!Worklist.empty() && --MaxLookup); + + // If we hit the maximum number of instructions to examine, be conservative. + if (!Worklist.empty()) + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + + return Result; +} + ModRefInfo BasicAAResult::getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { // Checking for known builtin intrinsics and target library functions. diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp --- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -525,7 +525,7 @@ } // Stores don't alias loads from read-only memory. - if (BatchAA.pointsToConstantMemory(LoadLoc)) + if (!isModSet(BatchAA.getModRefInfoMask(LoadLoc))) continue; // Stores depend on may/must aliased loads. diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp --- a/llvm/lib/Analysis/MemorySSA.cpp +++ b/llvm/lib/Analysis/MemorySSA.cpp @@ -378,9 +378,10 @@ const Instruction *I) { // If the memory can't be changed, then loads of the memory can't be // clobbered. - if (auto *LI = dyn_cast(I)) + if (auto *LI = dyn_cast(I)) { return I->hasMetadata(LLVMContext::MD_invariant_load) || - AA.pointsToConstantMemory(MemoryLocation::get(LI)); + !isModSet(AA.getModRefInfoMask(MemoryLocation::get(LI))); + } return false; } diff --git a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp --- a/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/llvm/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -385,23 +385,23 @@ return AliasResult::NoAlias; } -bool TypeBasedAAResult::pointsToConstantMemory(const MemoryLocation &Loc, - AAQueryInfo &AAQI, - bool OrLocal) { +ModRefInfo TypeBasedAAResult::getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) { if (!EnableTBAA) - return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); const MDNode *M = Loc.AATags.TBAA; if (!M) - return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); // If this is an "immutable" type, we can assume the pointer is pointing // to constant memory. if ((!isStructPathTBAA(M) && TBAANode(M).isTypeImmutable()) || (isStructPathTBAA(M) && TBAAStructTagNode(M).isTypeImmutable())) - return true; + return ModRefInfo::NoModRef; - return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); } MemoryEffects TypeBasedAAResult::getMemoryEffects(const CallBase *Call, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h --- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h @@ -38,8 +38,8 @@ AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, AAQueryInfo &AAQI); - bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, - bool OrLocal); + ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool IgnoreLocals); }; /// Analysis pass providing a never-invalidated alias analysis result. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp @@ -124,29 +124,33 @@ return AAResultBase::alias(LocA, LocB, AAQI); } -bool AMDGPUAAResult::pointsToConstantMemory(const MemoryLocation &Loc, - AAQueryInfo &AAQI, bool OrLocal) { +ModRefInfo AMDGPUAAResult::getModRefInfoMask(const MemoryLocation &Loc, + AAQueryInfo &AAQI, + bool IgnoreLocals) { + if (!Loc.Ptr) + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); + unsigned AS = Loc.Ptr->getType()->getPointerAddressSpace(); if (AS == AMDGPUAS::CONSTANT_ADDRESS || AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) - return true; + return ModRefInfo::NoModRef; const Value *Base = getUnderlyingObject(Loc.Ptr); AS = Base->getType()->getPointerAddressSpace(); if (AS == AMDGPUAS::CONSTANT_ADDRESS || AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) - return true; + return ModRefInfo::NoModRef; if (const GlobalVariable *GV = dyn_cast(Base)) { if (GV->isConstant()) - return true; + return ModRefInfo::NoModRef; } else if (const Argument *Arg = dyn_cast(Base)) { const Function *F = Arg->getParent(); // Only assume constant memory for arguments on kernels. switch (F->getCallingConv()) { default: - return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); case CallingConv::AMDGPU_LS: case CallingConv::AMDGPU_HS: case CallingConv::AMDGPU_ES: @@ -170,8 +174,8 @@ if (F->hasParamAttribute(ArgNo, Attribute::NoAlias) && (F->hasParamAttribute(ArgNo, Attribute::ReadNone) || F->hasParamAttribute(ArgNo, Attribute::ReadOnly))) { - return true; + return ModRefInfo::Ref; } } - return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); + return AAResultBase::getModRefInfoMask(Loc, AAQI, IgnoreLocals); } diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp --- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -140,13 +140,14 @@ ME |= MemoryEffects::argMemOnly(ModRefInfo::ModRef); auto AddLocAccess = [&](const MemoryLocation &Loc, ModRefInfo MR) { - // Ignore accesses to local memory. - if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true)) + // Ignore accesses to known-invariant or local memory. + MR &= AAR.getModRefInfoMask(Loc, /*IgnoreLocal=*/true); + if (isNoModRef(MR)) return; const Value *UO = getUnderlyingObject(Loc.Ptr); assert(!isa(UO) && - "Should have been handled by pointsToConstantMemory()"); + "Should have been handled by getModRefInfoMask()"); if (isa(UO)) { ME |= MemoryEffects::argMemOnly(MR); return; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -135,7 +135,7 @@ // If we have a store to a location which is known constant, we can conclude // that the store must be storing the constant value (else the memory // wouldn't be constant), and this must be a noop. - if (AA->pointsToConstantMemory(MI->getDest())) { + if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) { // Set the size of the copy to 0, it will be deleted on the next iteration. MI->setLength(Constant::getNullValue(MI->getLength()->getType())); return MI; @@ -252,7 +252,7 @@ // If we have a store to a location which is known constant, we can conclude // that the store must be storing the constant value (else the memory // wouldn't be constant), and this must be a noop. - if (AA->pointsToConstantMemory(MI->getDest())) { + if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) { // Set the size of the copy to 0, it will be deleted on the next iteration. MI->setLength(Constant::getNullValue(MI->getLength()->getType())); return MI; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -31,20 +31,20 @@ STATISTIC(NumDeadStore, "Number of dead stores eliminated"); STATISTIC(NumGlobalCopies, "Number of allocas copied from constant global"); -/// isOnlyCopiedFromConstantGlobal - Recursively walk the uses of a (derived) +/// isOnlyCopiedFromConstantMemory - Recursively walk the uses of a (derived) /// pointer to an alloca. Ignore any reads of the pointer, return false if we /// see any stores or other unknown uses. If we see pointer arithmetic, keep /// track of whether it moves the pointer (with IsOffset) but otherwise traverse /// the uses. If we see a memcpy/memmove that targets an unoffseted pointer to -/// the alloca, and if the source pointer is a pointer to a constant global, we -/// can optimize this. +/// the alloca, and if the source pointer is a pointer to a constant memory +/// location, we can optimize this. static bool isOnlyCopiedFromConstantMemory(AAResults *AA, Value *V, MemTransferInst *&TheCopy, SmallVectorImpl &ToDelete) { // We track lifetime intrinsics as we encounter them. If we decide to go - // ahead and replace the value with the global, this lets the caller quickly - // eliminate the markers. + // ahead and replace the value with the memory location, this lets the caller + // quickly eliminate the markers. SmallVector, 35> ValuesToInspect; ValuesToInspect.emplace_back(V, false); @@ -128,8 +128,8 @@ // If the memintrinsic isn't using the alloca as the dest, reject it. if (U.getOperandNo() != 0) return false; - // If the source of the memcpy/move is not a constant global, reject it. - if (!AA->pointsToConstantMemory(MI->getSource())) + // If the source of the memcpy/move is not constant, reject it. + if (isModSet(AA->getModRefInfoMask(MI->getSource()))) return false; // Otherwise, the transform is safe. Remember the copy instruction. @@ -139,9 +139,10 @@ return true; } -/// isOnlyCopiedFromConstantGlobal - Return true if the specified alloca is only -/// modified by a copy from a constant global. If we can prove this, we can -/// replace any uses of the alloca with uses of the global directly. +/// isOnlyCopiedFromConstantMemory - Return true if the specified alloca is only +/// modified by a copy from a constant memory location. If we can prove this, we +/// can replace any uses of the alloca with uses of the memory location +/// directly. static MemTransferInst * isOnlyCopiedFromConstantMemory(AAResults *AA, AllocaInst *AI, @@ -395,11 +396,11 @@ } // Check to see if this allocation is only modified by a memcpy/memmove from - // a constant whose alignment is equal to or exceeds that of the allocation. - // If this is the case, we can change all users to use the constant global - // instead. This is commonly produced by the CFE by constructs like "void - // foo() { int A[] = {1,2,3,4,5,6,7,8,9...}; }" if 'A' is only subsequently - // read. + // a memory location whose alignment is equal to or exceeds that of the + // allocation. If this is the case, we can change all users to use the + // constant memory location instead. This is commonly produced by the CFE by + // constructs like "void foo() { int A[] = {1,2,3,4,5,6,7,8,9...}; }" if 'A' + // is only subsequently read. SmallVector ToDelete; if (MemTransferInst *Copy = isOnlyCopiedFromConstantMemory(AA, &AI, ToDelete)) { Value *TheSrc = Copy->getSource(); @@ -1372,7 +1373,7 @@ // If we have a store to a location which is known constant, we can conclude // that the store must be storing the constant value (else the memory // wouldn't be constant), and this must be a noop. - if (AA->pointsToConstantMemory(Ptr)) + if (!isModSet(AA->getModRefInfoMask(Ptr))) return eraseInstFromFunction(SI); // Do really simple DSE, to catch cases where there are several consecutive diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -1161,7 +1161,7 @@ // Loads from constant memory are always safe to move, even if they end up // in the same alias set as something that ends up being modified. - if (AA->pointsToConstantMemory(LI->getOperand(0))) + if (!isModSet(AA->getModRefInfoMask(LI->getOperand(0)))) return true; if (LI->hasMetadata(LLVMContext::MD_invariant_load)) return true; diff --git a/llvm/lib/Transforms/Scalar/LoopPredication.cpp b/llvm/lib/Transforms/Scalar/LoopPredication.cpp --- a/llvm/lib/Transforms/Scalar/LoopPredication.cpp +++ b/llvm/lib/Transforms/Scalar/LoopPredication.cpp @@ -569,7 +569,7 @@ if (const SCEVUnknown *U = dyn_cast(S)) if (const auto *LI = dyn_cast(U->getValue())) if (LI->isUnordered() && L->hasLoopInvariantOperands(LI)) - if (AA->pointsToConstantMemory(LI->getOperand(0)) || + if (!isModSet(AA->getModRefInfoMask(LI->getOperand(0))) || LI->hasMetadata(LLVMContext::MD_invariant_load)) return true; return false; diff --git a/llvm/test/Analysis/BasicAA/constant-memory.ll b/llvm/test/Analysis/BasicAA/constant-memory.ll --- a/llvm/test/Analysis/BasicAA/constant-memory.ll +++ b/llvm/test/Analysis/BasicAA/constant-memory.ll @@ -4,18 +4,16 @@ declare void @dummy() -; FIXME: This could be NoModRef ; CHECK-LABEL: Function: basic -; CHECK: Just Ref: Ptr: i32* @c <-> call void @dummy() +; CHECK: NoModRef: Ptr: i32* @c <-> call void @dummy() define void @basic(ptr %p) { call void @dummy() load i32, ptr @c ret void } -; FIXME: This could be NoModRef ; CHECK-LABEL: Function: recphi -; CHECK: Just Ref: Ptr: i32* %p <-> call void @dummy() +; CHECK: NoModRef: Ptr: i32* %p <-> call void @dummy() define void @recphi() { entry: br label %loop diff --git a/llvm/test/Analysis/BasicAA/pr18573.ll b/llvm/test/Analysis/BasicAA/pr18573.ll --- a/llvm/test/Analysis/BasicAA/pr18573.ll +++ b/llvm/test/Analysis/BasicAA/pr18573.ll @@ -8,7 +8,7 @@ declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float>, i8*, <8 x i32>, <8 x float>, i8) #0 ; Function Attrs: nounwind -define <8 x float> @foo1(i8* noalias readonly %arr.ptr, <8 x i32>* noalias readonly %vix.ptr, i8* noalias %t2.ptr) #1 { +define <8 x float> @foo1(i8* noalias %arr.ptr, <8 x i32>* noalias readonly %vix.ptr, i8* noalias %t2.ptr) #1 { allocas: %vix = load <8 x i32>, <8 x i32>* %vix.ptr, align 4 %t1.ptr = getelementptr i8, i8* %arr.ptr, i8 4 diff --git a/llvm/test/Analysis/BasicAA/readonly-noalias.ll b/llvm/test/Analysis/BasicAA/readonly-noalias.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Analysis/BasicAA/readonly-noalias.ll @@ -0,0 +1,13 @@ +; Tests that readonly noalias implies !Mod. +; +; RUN: opt < %s -passes=aa-eval -print-all-alias-modref-info 2>&1 | FileCheck %s + +declare void @foo(ptr) + +; CHECK-LABEL: Function: basic +; CHECK: Just Ref: Ptr: i32* %p <-> call void @foo(ptr %p) +define void @basic(ptr readonly noalias %p) { + call void @foo(ptr %p) + load i32, ptr %p + ret void +} diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/aliasing.ll b/llvm/test/CodeGen/ARM/ParallelDSP/aliasing.ll --- a/llvm/test/CodeGen/ARM/ParallelDSP/aliasing.ll +++ b/llvm/test/CodeGen/ARM/ParallelDSP/aliasing.ll @@ -4,6 +4,7 @@ ; instruction possibly aliasing any mul load operands; arguments are passed ; without 'restrict' enabled. ; +; CHECK-LABEL: @no_restrict ; CHECK-NOT: call i32 @llvm.arm.smlad ; define dso_local i32 @no_restrict(i32 %arg, i32* nocapture %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) { @@ -52,9 +53,10 @@ ; aliasing one of the mul load operands. Arguments are now annotated with ; 'noalias'. ; +; CHECK-LABEL: @restrict ; CHECK-NOT: call i32 @llvm.arm.smlad ; -define dso_local i32 @restrict(i32 %arg, i32* noalias %arg1, i16* noalias readonly %arg2, i16* noalias readonly %arg3) { +define dso_local i32 @restrict(i32 %arg, i32* noalias %arg1, i16* noalias readonly %arg2, i16* noalias %arg3) { entry: %cmp24 = icmp sgt i32 %arg, 0 br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup @@ -289,7 +291,7 @@ ; CHECK-LABEL: store_alias_arg3_illegal_1 ; CHECK-NOT: load i32 -define dso_local i32 @store_alias_arg3_illegal_1(i32 %arg, i32* nocapture %arg1, i16* noalias nocapture readonly %arg2, i16* noalias nocapture readonly %arg3) { +define dso_local i32 @store_alias_arg3_illegal_1(i32 %arg, i32* nocapture %arg1, i16* noalias nocapture readonly %arg2, i16* noalias nocapture %arg3) { entry: %cmp24 = icmp sgt i32 %arg, 0 br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup @@ -330,7 +332,7 @@ ; CHECK-LABEL: store_alias_arg3_illegal_2 ; CHECK-NOT: load i32 -define dso_local i32 @store_alias_arg3_illegal_2(i32 %arg, i32* nocapture %arg1, i16* noalias nocapture readonly %arg2, i16* noalias nocapture readonly %arg3) { +define dso_local i32 @store_alias_arg3_illegal_2(i32 %arg, i32* nocapture %arg1, i16* noalias nocapture readonly %arg2, i16* noalias nocapture %arg3) { entry: %cmp24 = icmp sgt i32 %arg, 0 br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup @@ -455,7 +457,7 @@ ; when it finds the alias. ; CHECK-LABEL: one_pair_alias ; CHECK-NOT: call i32 @llvm.arm.smlad -define i32 @one_pair_alias(i16* noalias nocapture readonly %b, i16* noalias nocapture readonly %c) { +define i32 @one_pair_alias(i16* noalias nocapture readonly %b, i16* noalias nocapture %c) { entry: br label %for.body diff --git a/llvm/test/Transforms/InstCombine/copied-from-readonly-noalias.ll b/llvm/test/Transforms/InstCombine/copied-from-readonly-noalias.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/copied-from-readonly-noalias.ll @@ -0,0 +1,19 @@ +; Tests that we can eliminate allocas copied from readonly noalias pointers. +; +; RUN: opt -S < %s -aa-pipeline=basic-aa -passes=instcombine 2>&1 | FileCheck %s + +%struct.BigStruct = type { i32, i32, i32 } + +declare void @bar(ptr readonly noalias noundef nocapture) readnone + +declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) + +; CHECK-LABEL: define void @foo +; CHECK-NEXT: call void @bar +; CHECK-NEXT: ret void +define void @foo(ptr nocapture noundef readonly noalias align(4) dereferenceable(12) %arg) { + %alloca = alloca %struct.BigStruct, align 4 + call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(12) %alloca, ptr noundef nonnull align 4 dereferenceable(12) %arg, i64 12, i1 false) + call void @bar(ptr readonly noalias noundef nonnull nocapture %alloca) readnone + ret void +} diff --git a/llvm/test/Transforms/InstCombine/store.ll b/llvm/test/Transforms/InstCombine/store.ll --- a/llvm/test/Transforms/InstCombine/store.ll +++ b/llvm/test/Transforms/InstCombine/store.ll @@ -336,6 +336,14 @@ ret void } +; Delete stores to readonly noalias pointers. +define void @store_to_readonly_noalias(ptr readonly noalias %0) { +; CHECK-LABEL: @store_to_readonly_noalias( +; CHECK-NEXT: ret void + store i32 3, ptr %0, align 4 + ret void +} + !0 = !{!4, !4, i64 0} !1 = !{!"omnipotent char", !2} !2 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll @@ -292,7 +292,7 @@ -define void @gather_nxv4i32_ind64_stride2(float* noalias nocapture readonly %a, float* noalias nocapture readonly %b, i64 %n) #0 { +define void @gather_nxv4i32_ind64_stride2(float* noalias nocapture %a, float* noalias nocapture readonly %b, i64 %n) #0 { ; CHECK-LABEL: @gather_nxv4i32_ind64_stride2( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll --- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll @@ -572,15 +572,15 @@ ; void load_gap_reverse(struct pair *P1, struct pair *P2, int X) { ; for (int i = 1023; i >= 0; i--) { ; int a = X + i; -; int b = A[i].y - i; -; B[i].x = a; +; int b = B[i].y - i; +; A[i].x = a; ; B[i].y = b; ; } ; } %pair = type { i64, i64 } -define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noalias nocapture readonly %P2, i64 %X) { +define void @load_gap_reverse(%pair* noalias nocapture %P1, %pair* noalias nocapture %P2, i64 %X) { ; CHECK-LABEL: @load_gap_reverse( ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]