diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -212,46 +212,45 @@ /// Helper to represent an access offset and size, with logic to deal with /// uncertainty and check for overlapping accesses. -struct OffsetAndSize { +struct RangeTy { int64_t Offset = Unassigned; int64_t Size = Unassigned; - OffsetAndSize(int64_t Offset, int64_t Size) : Offset(Offset), Size(Size) {} - OffsetAndSize() = default; - static OffsetAndSize getUnknown() { return OffsetAndSize{Unknown, Unknown}; } + RangeTy(int64_t Offset, int64_t Size) : Offset(Offset), Size(Size) {} + RangeTy() = default; + static RangeTy getUnknown() { return RangeTy{Unknown, Unknown}; } /// Return true if offset or size are unknown. bool offsetOrSizeAreUnknown() const { - return Offset == OffsetAndSize::Unknown || Size == OffsetAndSize::Unknown; + return Offset == RangeTy::Unknown || Size == RangeTy::Unknown; } /// Return true if offset and size are unknown, thus this is the default /// unknown object. bool offsetAndSizeAreUnknown() const { - return Offset == OffsetAndSize::Unknown && Size == OffsetAndSize::Unknown; + return Offset == RangeTy::Unknown && Size == RangeTy::Unknown; } /// Return true if the offset and size are unassigned. bool isUnassigned() const { - assert((Offset == OffsetAndSize::Unassigned) == - (Size == OffsetAndSize::Unassigned) && + assert((Offset == RangeTy::Unassigned) == (Size == RangeTy::Unassigned) && "Inconsistent state!"); - return Offset == OffsetAndSize::Unassigned; + return Offset == RangeTy::Unassigned; } /// Return true if this offset and size pair might describe an address that - /// overlaps with \p OAS. - bool mayOverlap(const OffsetAndSize &OAS) const { + /// overlaps with \p Range. + bool mayOverlap(const RangeTy &Range) const { // Any unknown value and we are giving up -> overlap. - if (offsetOrSizeAreUnknown() || OAS.offsetOrSizeAreUnknown()) + if (offsetOrSizeAreUnknown() || Range.offsetOrSizeAreUnknown()) return true; // Check if one offset point is in the other interval [offset, // offset+size]. - return OAS.Offset + OAS.Size > Offset && OAS.Offset < Offset + Size; + return Range.Offset + Range.Size > Offset && Range.Offset < Offset + Size; } - OffsetAndSize &operator&=(const OffsetAndSize &R) { + RangeTy &operator&=(const RangeTy &R) { if (Offset == Unassigned) Offset = R.Offset; else if (R.Offset != Unassigned && R.Offset != Offset) @@ -275,19 +274,17 @@ static constexpr int64_t Unknown = -2; }; -inline bool operator==(const OffsetAndSize &A, const OffsetAndSize &B) { +inline bool operator==(const RangeTy &A, const RangeTy &B) { return A.Offset == B.Offset && A.Size == B.Size; } -inline bool operator!=(const OffsetAndSize &A, const OffsetAndSize &B) { - return !(A == B); -} +inline bool operator!=(const RangeTy &A, const RangeTy &B) { return !(A == B); } /// Return the initial value of \p Obj with type \p Ty if that is a constant. Constant *getInitialValueForObj(Value &Obj, Type &Ty, const TargetLibraryInfo *TLI, const DataLayout &DL, - OffsetAndSize *OASPtr = nullptr); + RangeTy *RangePtr = nullptr); /// Collect all potential underlying objects of \p Ptr at position \p CtxI in /// \p Objects. Assumed information is used and dependences onto \p QueryingAA @@ -5013,25 +5010,25 @@ struct Access { Access(Instruction *I, int64_t Offset, int64_t Size, std::optional Content, AccessKind Kind, Type *Ty) - : LocalI(I), RemoteI(I), Content(Content), OAS(Offset, Size), + : LocalI(I), RemoteI(I), Content(Content), Range(Offset, Size), Kind(Kind), Ty(Ty) { verify(); } Access(Instruction *LocalI, Instruction *RemoteI, int64_t Offset, int64_t Size, std::optional Content, AccessKind Kind, Type *Ty) - : LocalI(LocalI), RemoteI(RemoteI), Content(Content), OAS(Offset, Size), - Kind(Kind), Ty(Ty) { + : LocalI(LocalI), RemoteI(RemoteI), Content(Content), + Range(Offset, Size), Kind(Kind), Ty(Ty) { verify(); } Access(const Access &Other) = default; Access(const Access &&Other) : LocalI(Other.LocalI), RemoteI(Other.RemoteI), Content(Other.Content), - OAS(Other.OAS), Kind(Other.Kind), Ty(Other.Ty) {} + Range(Other.Range), Kind(Other.Kind), Ty(Other.Ty) {} Access &operator=(const Access &Other) = default; bool operator==(const Access &R) const { - return LocalI == R.LocalI && RemoteI == R.RemoteI && OAS == R.OAS && + return LocalI == R.LocalI && RemoteI == R.RemoteI && Range == R.Range && Content == R.Content && Kind == R.Kind; } bool operator!=(const Access &R) const { return !(*this == R); } @@ -5040,13 +5037,13 @@ assert(RemoteI == R.RemoteI && "Expected same instruction!"); assert(LocalI == R.LocalI && "Expected same instruction!"); Kind = AccessKind(Kind | R.Kind); - auto Before = OAS; - OAS &= R.OAS; - if (Before.isUnassigned() || Before == OAS) { + auto Before = Range; + Range &= R.Range; + if (Before.isUnassigned() || Before == Range) { Content = AA::combineOptionalValuesInAAValueLatice(Content, R.Content, Ty); } else { - // Since the OAS information changed, set a conservative state -- drop + // Since the Range information changed, set a conservative state -- drop // the contents, and assume MayAccess rather than MustAccess. setWrittenValueUnknown(); Kind = AccessKind(Kind | AK_MAY); @@ -5114,10 +5111,10 @@ std::optional getContent() const { return Content; } /// Return the offset for this access. - int64_t getOffset() const { return OAS.Offset; } + int64_t getOffset() const { return Range.Offset; } /// Return the size for this access. - int64_t getSize() const { return OAS.Size; } + int64_t getSize() const { return Range.Size; } private: /// The instruction responsible for the access with respect to the local @@ -5132,7 +5129,7 @@ std::optional Content; /// The object accessed, in terms of an offset and size in bytes. - AA::OffsetAndSize OAS; + AA::RangeTy Range; /// The access kind, e.g., READ, as bitset (could be more than one). AccessKind Kind; @@ -5151,13 +5148,12 @@ /// See AbstractAttribute::getIdAddr() const char *getIdAddr() const override { return &ID; } - /// Call \p CB on all accesses that might interfere with \p OAS and return + /// Call \p CB on all accesses that might interfere with \p Range and return /// true if all such accesses were known and the callback returned true for /// all of them, false otherwise. An access interferes with an offset-size /// pair if it might read or write that memory region. virtual bool forallInterferingAccesses( - AA::OffsetAndSize OAS, - function_ref CB) const = 0; + AA::RangeTy Range, function_ref CB) const = 0; /// Call \p CB on all accesses that might interfere with \p I and /// return true if all such accesses were known and the callback returned true @@ -5169,7 +5165,7 @@ virtual bool forallInterferingAccesses( Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I, function_ref CB, bool &HasBeenWrittenTo, - AA::OffsetAndSize &OAS) const = 0; + AA::RangeTy &Range) const = 0; /// This function should return true if the type of the \p AA is AAPointerInfo static bool classof(const AbstractAttribute *AA) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -547,32 +547,31 @@ bool funcRetrievesMultigridSyncArg(Attributor &A) { auto Pos = llvm::AMDGPU::getMultigridSyncArgImplicitArgPosition(); - AA::OffsetAndSize OAS(Pos, 8); - return funcRetrievesImplicitKernelArg(A, OAS); + AA::RangeTy Range(Pos, 8); + return funcRetrievesImplicitKernelArg(A, Range); } bool funcRetrievesHostcallPtr(Attributor &A) { auto Pos = llvm::AMDGPU::getHostcallImplicitArgPosition(); - AA::OffsetAndSize OAS(Pos, 8); - return funcRetrievesImplicitKernelArg(A, OAS); + AA::RangeTy Range(Pos, 8); + return funcRetrievesImplicitKernelArg(A, Range); } bool funcRetrievesHeapPtr(Attributor &A) { if (AMDGPU::getAmdhsaCodeObjectVersion() != 5) return false; - AA::OffsetAndSize OAS(AMDGPU::ImplicitArg::HEAP_PTR_OFFSET, 8); - return funcRetrievesImplicitKernelArg(A, OAS); + AA::RangeTy Range(AMDGPU::ImplicitArg::HEAP_PTR_OFFSET, 8); + return funcRetrievesImplicitKernelArg(A, Range); } bool funcRetrievesQueuePtr(Attributor &A) { if (AMDGPU::getAmdhsaCodeObjectVersion() != 5) return false; - AA::OffsetAndSize OAS(AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET, 8); - return funcRetrievesImplicitKernelArg(A, OAS); + AA::RangeTy Range(AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET, 8); + return funcRetrievesImplicitKernelArg(A, Range); } - bool funcRetrievesImplicitKernelArg(Attributor &A, - AA::OffsetAndSize OAS) { + bool funcRetrievesImplicitKernelArg(Attributor &A, AA::RangeTy Range) { // Check if this is a call to the implicitarg_ptr builtin and it // is used to retrieve the hostcall pointer. The implicit arg for // hostcall is not used only if every use of the implicitarg_ptr @@ -588,7 +587,7 @@ *this, IRPosition::callsite_returned(Call), DepClassTy::REQUIRED); return PointerInfoAA.forallInterferingAccesses( - OAS, [](const AAPointerInfo::Access &Acc, bool IsExact) { + Range, [](const AAPointerInfo::Access &Acc, bool IsExact) { return Acc.getRemoteInst()->isDroppable(); }); }; diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp --- a/llvm/lib/Transforms/IPO/Attributor.cpp +++ b/llvm/lib/Transforms/IPO/Attributor.cpp @@ -226,7 +226,7 @@ Constant *AA::getInitialValueForObj(Value &Obj, Type &Ty, const TargetLibraryInfo *TLI, const DataLayout &DL, - AA::OffsetAndSize *OASPtr) { + AA::RangeTy *RangePtr) { if (isa(Obj)) return UndefValue::get(&Ty); if (Constant *Init = getInitialValueOfAllocation(&Obj, TLI, &Ty)) @@ -239,8 +239,8 @@ if (!GV->hasInitializer()) return UndefValue::get(&Ty); - if (OASPtr && !OASPtr->offsetOrSizeAreUnknown()) { - APInt Offset = APInt(64, OASPtr->Offset); + if (RangePtr && !RangePtr->offsetOrSizeAreUnknown()) { + APInt Offset = APInt(64, RangePtr->Offset); return ConstantFoldLoadFromConst(GV->getInitializer(), &Ty, Offset, DL); } @@ -456,11 +456,11 @@ // object. bool HasBeenWrittenTo = false; - AA::OffsetAndSize OAS; + AA::RangeTy Range; auto &PI = A.getAAFor(QueryingAA, IRPosition::value(*Obj), DepClassTy::NONE); if (!PI.forallInterferingAccesses(A, QueryingAA, I, CheckAccess, - HasBeenWrittenTo, OAS)) { + HasBeenWrittenTo, Range)) { LLVM_DEBUG( dbgs() << "Failed to verify all interfering accesses for underlying object: " @@ -468,10 +468,10 @@ return false; } - if (IsLoad && !HasBeenWrittenTo && !OAS.isUnassigned()) { + if (IsLoad && !HasBeenWrittenTo && !Range.isUnassigned()) { const DataLayout &DL = A.getDataLayout(); Value *InitialValue = - AA::getInitialValueForObj(*Obj, *I.getType(), TLI, DL, &OAS); + AA::getInitialValueForObj(*Obj, *I.getType(), TLI, DL, &Range); if (!InitialValue) { LLVM_DEBUG(dbgs() << "Could not determine required initial value of " "underlying object, abort!\n"); diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -720,25 +720,25 @@ static bool isEqual(const Access &LHS, const Access &RHS); }; -/// Helper that allows OffsetAndSize as a key in a DenseMap. -template <> struct DenseMapInfo { - static inline AA::OffsetAndSize getEmptyKey() { +/// Helper that allows RangeTy as a key in a DenseMap. +template <> struct DenseMapInfo { + static inline AA::RangeTy getEmptyKey() { auto EmptyKey = DenseMapInfo::getEmptyKey(); - return AA::OffsetAndSize{EmptyKey, EmptyKey}; + return AA::RangeTy{EmptyKey, EmptyKey}; } - static inline AA::OffsetAndSize getTombstoneKey() { + static inline AA::RangeTy getTombstoneKey() { auto TombstoneKey = DenseMapInfo::getTombstoneKey(); - return AA::OffsetAndSize{TombstoneKey, TombstoneKey}; + return AA::RangeTy{TombstoneKey, TombstoneKey}; } - static unsigned getHashValue(const AA::OffsetAndSize &OAS) { + static unsigned getHashValue(const AA::RangeTy &Range) { return detail::combineHashValue( - DenseMapInfo::getHashValue(OAS.Offset), - DenseMapInfo::getHashValue(OAS.Size)); + DenseMapInfo::getHashValue(Range.Offset), + DenseMapInfo::getHashValue(Range.Size)); } - static bool isEqual(const AA::OffsetAndSize &A, const AA::OffsetAndSize B) { + static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) { return A == B; } }; @@ -824,7 +824,7 @@ AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI = nullptr); - using OffsetBinsTy = DenseMap>; + using OffsetBinsTy = DenseMap>; using const_bin_iterator = OffsetBinsTy::const_iterator; const_bin_iterator begin() const { return OffsetBins.begin(); } @@ -838,7 +838,7 @@ // Every memory instruction results in an Access object. We maintain a list of // all Access objects that we own, along with the following maps: // - // - OffsetBins: OffsetAndSize -> { Access } + // - OffsetBins: RangeTy -> { Access } // - RemoteIMap: RemoteI x LocalI -> Access // // A RemoteI is any instruction that accesses memory. RemoteI is different @@ -846,23 +846,23 @@ // instruction in the callgraph starting from LocalI. Multiple paths in the // callgraph from LocalI to RemoteI may produce multiple accesses, but these // are all combined into a single Access object. This may result in loss of - // information in OffsetAndSize in the Access object. + // information in RangeTy in the Access object. SmallVector AccessList; OffsetBinsTy OffsetBins; DenseMap> RemoteIMap; /// See AAPointerInfo::forallInterferingAccesses. bool forallInterferingAccesses( - AA::OffsetAndSize OAS, + AA::RangeTy Range, function_ref CB) const { if (!isValidState()) return false; for (const auto &It : OffsetBins) { - AA::OffsetAndSize ItOAS = It.getFirst(); - if (!OAS.mayOverlap(ItOAS)) + AA::RangeTy ItRange = It.getFirst(); + if (!Range.mayOverlap(ItRange)) continue; - bool IsExact = OAS == ItOAS && !OAS.offsetOrSizeAreUnknown(); + bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown(); for (auto Index : It.getSecond()) { auto &Access = AccessList[Index]; if (!CB(Access, IsExact)) @@ -876,7 +876,7 @@ bool forallInterferingAccesses( Instruction &I, function_ref CB, - AA::OffsetAndSize &OAS) const { + AA::RangeTy &Range) const { if (!isValidState()) return false; @@ -887,9 +887,9 @@ for (auto LI : LocalList->getSecond()) { auto &Access = AccessList[LI]; - OAS &= {Access.getOffset(), Access.getSize()}; + Range &= {Access.getOffset(), Access.getSize()}; } - return forallInterferingAccesses(OAS, CB); + return forallInterferingAccesses(Range, CB); } private: @@ -929,7 +929,7 @@ return ChangeStatus::UNCHANGED; Acc = Current; - AA::OffsetAndSize Key{Before.getOffset(), Before.getSize()}; + AA::RangeTy Key{Before.getOffset(), Before.getSize()}; assert(OffsetBins.count(Key) && "Existing Access must be in some bin."); auto &Bin = OffsetBins[Key]; assert(Bin.count(AccIndex) && @@ -940,7 +940,7 @@ Bin.erase(AccIndex); } - AA::OffsetAndSize Key{Acc.getOffset(), Acc.getSize()}; + AA::RangeTy Key{Acc.getOffset(), Acc.getSize()}; LLVM_DEBUG(dbgs() << "[AAPointerInfo] Inserting Access " << Acc << " with key {" << Key.Offset << ',' << Key.Size << "}\n"); OffsetBins[Key].insert(AccIndex); @@ -967,16 +967,16 @@ } bool forallInterferingAccesses( - AA::OffsetAndSize OAS, + AA::RangeTy Range, function_ref CB) const override { - return State::forallInterferingAccesses(OAS, CB); + return State::forallInterferingAccesses(Range, CB); } bool forallInterferingAccesses( Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I, function_ref UserCB, bool &HasBeenWrittenTo, - AA::OffsetAndSize &OAS) const override { + AA::RangeTy &Range) const override { HasBeenWrittenTo = false; SmallPtrSet DominatingWrites; @@ -1091,7 +1091,7 @@ InterferingAccesses.push_back({&Acc, Exact}); return true; }; - if (!State::forallInterferingAccesses(I, AccessCB, OAS)) + if (!State::forallInterferingAccesses(I, AccessCB, Range)) return false; if (HasBeenWrittenTo) { @@ -1160,10 +1160,10 @@ ChangeStatus Changed = ChangeStatus::UNCHANGED; const auto &State = OtherAAImpl.getState(); for (const auto &It : State) { - AA::OffsetAndSize OAS = AA::OffsetAndSize::getUnknown(); - if (Offset != AA::OffsetAndSize::Unknown && + AA::RangeTy Range = AA::RangeTy::getUnknown(); + if (Offset != AA::RangeTy::Unknown && !It.first.offsetOrSizeAreUnknown()) { - OAS = AA::OffsetAndSize(It.first.Offset + Offset, It.first.Size); + Range = AA::RangeTy(It.first.Offset + Offset, It.first.Size); } for (auto Index : It.getSecond()) { const auto &RAcc = State.getAccess(Index); @@ -1179,8 +1179,8 @@ AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW)); AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST)); } - Changed = Changed | addAccess(A, OAS.Offset, OAS.Size, CB, Content, AK, - RAcc.getType(), RAcc.getRemoteInst()); + Changed = Changed | addAccess(A, Range.Offset, Range.Size, CB, Content, + AK, RAcc.getType(), RAcc.getRemoteInst()); } } return Changed; @@ -1221,10 +1221,10 @@ bool handleAccess(Attributor &A, Instruction &I, Value &Ptr, std::optional Content, AccessKind Kind, int64_t Offset, ChangeStatus &Changed, Type *Ty, - int64_t Size = AA::OffsetAndSize::Unknown) { + int64_t Size = AA::RangeTy::Unknown) { using namespace AA::PointerInfo; // No need to find a size if one is given. - if (Size == AA::OffsetAndSize::Unknown && Ty) { + if (Size == AA::RangeTy::Unknown && Ty) { const DataLayout &DL = A.getDataLayout(); TypeSize AccessSize = DL.getTypeStoreSize(Ty); if (!AccessSize.isScalable()) @@ -1235,8 +1235,11 @@ }; /// Helper struct, will support ranges eventually. + /// + /// FIXME: Tracks a single Offset until we have proper support for a list of + /// RangeTy objects. struct OffsetInfo { - int64_t Offset = AA::OffsetAndSize::Unassigned; + int64_t Offset = AA::RangeTy::Unassigned; bool operator==(const OffsetInfo &OI) const { return Offset == OI.Offset; } }; @@ -1253,7 +1256,7 @@ auto HandlePassthroughUser = [&](Value *Usr, OffsetInfo PtrOI, bool &Follow) { - assert(PtrOI.Offset != AA::OffsetAndSize::Unassigned && + assert(PtrOI.Offset != AA::RangeTy::Unassigned && "Cannot pass through if the input Ptr was not visited!"); OffsetInfo &UsrOI = OffsetInfoMap[Usr]; UsrOI = PtrOI; @@ -1293,11 +1296,11 @@ // TODO: Use range information. APInt GEPOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0); - if (PtrOI.Offset == AA::OffsetAndSize::Unknown || + if (PtrOI.Offset == AA::RangeTy::Unknown || !GEP->accumulateConstantOffset(DL, GEPOffset)) { LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset not constant " << *GEP << "\n"); - UsrOI.Offset = AA::OffsetAndSize::Unknown; + UsrOI.Offset = AA::RangeTy::Unknown; Follow = true; return true; } @@ -1325,17 +1328,17 @@ // Check if the PHI operand has already an unknown offset as we can't // improve on that anymore. - if (PtrOI.Offset == AA::OffsetAndSize::Unknown) { + if (PtrOI.Offset == AA::RangeTy::Unknown) { LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown " << *CurPtr << " in " << *Usr << "\n"); - Follow = UsrOI.Offset != AA::OffsetAndSize::Unknown; + Follow = UsrOI.Offset != AA::RangeTy::Unknown; UsrOI = PtrOI; return true; } // Check if the PHI is invariant (so far). if (UsrOI == PtrOI) { - assert(PtrOI.Offset != AA::OffsetAndSize::Unassigned && + assert(PtrOI.Offset != AA::RangeTy::Unassigned && "Cannot assign if the current Ptr was not visited!"); LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)"); return true; @@ -1362,7 +1365,7 @@ // TODO: Approximate in case we know the direction of the recurrence. UsrOI = PtrOI; - UsrOI.Offset = AA::OffsetAndSize::Unknown; + UsrOI.Offset = AA::RangeTy::Unknown; Follow = true; return true; } @@ -1606,7 +1609,7 @@ // accessed. if (auto *MI = dyn_cast_or_null(getCtxI())) { ConstantInt *Length = dyn_cast(MI->getLength()); - int64_t LengthVal = AA::OffsetAndSize::Unknown; + int64_t LengthVal = AA::RangeTy::Unknown; if (Length) LengthVal = Length->getSExtValue(); Value &Ptr = getAssociatedValue(); @@ -1663,8 +1666,7 @@ handleAccess(A, *getCtxI(), getAssociatedValue(), nullptr, ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE, - AA::OffsetAndSize::Unknown, Changed, nullptr, - AA::OffsetAndSize::Unknown); + AA::RangeTy::Unknown, Changed, nullptr, AA::RangeTy::Unknown); return Changed; }