diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h --- a/llvm/include/llvm/Transforms/IPO/Attributor.h +++ b/llvm/include/llvm/Transforms/IPO/Attributor.h @@ -349,6 +349,32 @@ /// respective scope. virtual bool isAssumedWriteOnly() const = 0; + /// Return true if we know that the underlying value will only access + /// inaccesible memory only (see Attribute::InaccessibleMemOnly). + virtual bool isKnownInaccessibleMemOnly() const = 0; + + /// Return true if we assume that the underlying value will only access + /// inaccesible memory only (see Attribute::InaccessibleMemOnly). + virtual bool isAssumedInaccessibleMemOnly() const = 0; + + /// Return true if we know that the underlying value will only access + /// argument pointees (see Attribute::ArgMemOnly). + virtual bool isKnownArgMemOnly() const = 0; + + /// Return true if we assume that the underlying value will only access + /// argument pointees (see Attribute::ArgMemOnly). + virtual bool isAssumedArgMemOnly() const = 0; + + /// Return true if we know that the underlying value will only access + /// inaccesible memory or argument pointees (see + /// Attribute::InaccessibleOrArgMemOnly). + virtual bool isKnownInaccessibleOrArgMemOnly() const = 0; + + /// Return true if we assume that the underlying value will only access + /// inaccesible memory or argument pointees (see + /// Attribute::InaccessibleOrArgMemOnly). + virtual bool isAssumedInaccessibleOrArgMemOnly() const = 0; + /// See AbstractState::getAttrKind(). Attribute::AttrKind getAttrKind() const override { return ID; } diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp --- a/llvm/lib/Transforms/IPO/Attributor.cpp +++ b/llvm/lib/Transforms/IPO/Attributor.cpp @@ -72,6 +72,11 @@ STATISTIC(NumFnReadNone, "Number of functions marked read-none"); STATISTIC(NumFnReadOnly, "Number of functions marked read-only"); STATISTIC(NumFnWriteOnly, "Number of functions marked write-only"); +STATISTIC(NumFnArgMemOnly, "Number of functions marked argumen-pointee-only"); +STATISTIC(NumFnInaccessibleMemOnly, + "Number of functions marked inaccessible-memory-only"); +STATISTIC(NumFnInaccessibleOrArgMemOnly, + "Number of functions marked inaccessible-or-argument-pointee-only"); // TODO: Determine a good default value. // @@ -221,6 +226,15 @@ MP == AbstractAttribute::MP_ARGUMENT ? NumFnArgumentWriteOnly++ : NumFnWriteOnly++; return; + case Attribute::InaccessibleMemOnly: + NumFnInaccessibleMemOnly++; + return; + case Attribute::ArgMemOnly: + NumFnArgMemOnly++; + return; + case Attribute::InaccessibleMemOrArgMemOnly: + NumFnInaccessibleOrArgMemOnly++; + return; default: return; } @@ -1347,7 +1361,8 @@ } /// -------------------- Memory Behavior Attributes ---------------------------- -/// Includes read-none, read-only, and write-only. +/// Includes read-none, read-only, and write-only, as well as their combination +/// with inaccessible-mem-only, arg-mem-only, and inaccessible-or-arg-mem-only. /// ---------------------------------------------------------------------------- /// A class to hold the state of for memory behaviour attributes. @@ -1368,7 +1383,13 @@ NO_WRITES = 1 << 1, NO_ACCESSES = NO_READS | NO_WRITES, - BEST_STATE = NO_ACCESSES, + ONLY_ARGUMENT_MEM = 1 << 2, + ONLY_INACCESSIBLE_MEM = 1 << 3, + ONLY_INACCESSIBLE_OR_ARGUMENT_MEM = 1 << 4, + NO_LOCATIONS = ONLY_ARGUMENT_MEM | ONLY_INACCESSIBLE_MEM | + ONLY_INACCESSIBLE_OR_ARGUMENT_MEM, + + BEST_STATE = NO_ACCESSES | NO_LOCATIONS, }; /// See AAMemoryBehavior::isKnownReadNone(); @@ -1395,6 +1416,36 @@ return isAssumed(NO_READS); } + /// See AAMemoryBehavior::isKnownInaccessibleMemOnly(); + virtual bool isKnownInaccessibleMemOnly() const override { + return getKnown() & ONLY_INACCESSIBLE_MEM; + } + + /// See AAMemoryBehavior::isAssumedInaccessibleMemOnly(); + virtual bool isAssumedInaccessibleMemOnly() const override { + return getAssumed() & ONLY_INACCESSIBLE_MEM; + } + + /// See AAMemoryBehavior::isKnownArgMemOnly(); + virtual bool isKnownArgMemOnly() const override { + return getKnown() & ONLY_ARGUMENT_MEM; + } + + /// See AAMemoryBehavior::isAssumedArgMemOnly(); + virtual bool isAssumedArgMemOnly() const override { + return getAssumed() & ONLY_ARGUMENT_MEM; + } + + /// See AAMemoryBehavior::isKnownInaccessibleOrArgMemOnly(); + virtual bool isKnownInaccessibleOrArgMemOnly() const override { + return getKnown() & ONLY_INACCESSIBLE_OR_ARGUMENT_MEM; + } + + /// See AAMemoryBehavior::isAssumedInaccessibleOrArgMemOnly(); + virtual bool isAssumedInaccessibleOrArgMemOnly() const override { + return getAssumed() & ONLY_INACCESSIBLE_OR_ARGUMENT_MEM; + } + /// See AbstractState::getAsStr(). const std::string getAsStr() const override { std::string S = ""; @@ -1407,6 +1458,13 @@ else S = "may-read/write"; + if (isAssumedInaccessibleMemOnly()) + S += " inaccessible memory"; + else if (isAssumedArgMemOnly()) + S += " argument pointee memory"; + else if (isAssumedInaccessibleOrArgMemOnly()) + S += " inaccessible or argument pointee memory"; + return S; } @@ -1449,6 +1507,12 @@ State.addKnownBits(NO_WRITES); if (AAResults::doesNotReadMemory(ModRef)) State.addKnownBits(NO_READS); + if (AAResults::onlyAccessesInaccessibleMem(ModRef)) + State.addKnownBits(ONLY_INACCESSIBLE_MEM); + if (AAResults::onlyAccessesArgPointees(ModRef)) + State.addKnownBits(ONLY_ARGUMENT_MEM); + if (AAResults::onlyAccessesInaccessibleOrArgMem(ModRef)) + State.addKnownBits(ONLY_INACCESSIBLE_OR_ARGUMENT_MEM); } else { if (Fn->hasFnAttribute(Attribute::ReadNone)) State.addKnownBits(NO_ACCESSES); @@ -1456,6 +1520,12 @@ State.addKnownBits(NO_WRITES); else if (Fn->hasFnAttribute(Attribute::WriteOnly)) State.addKnownBits(NO_READS); + if (Fn->hasFnAttribute(Attribute::InaccessibleMemOnly)) + State.addKnownBits(ONLY_INACCESSIBLE_MEM); + else if (Fn->hasFnAttribute(Attribute::ArgMemOnly)) + State.addKnownBits(ONLY_ARGUMENT_MEM); + else if (Fn->hasFnAttribute(Attribute::InaccessibleMemOrArgMemOnly)) + State.addKnownBits(ONLY_INACCESSIBLE_OR_ARGUMENT_MEM); } } } @@ -1474,6 +1544,15 @@ else if (isAssumedWriteOnly()) Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly)); assert(Attrs.size() <= 1); + + if (isAssumedInaccessibleMemOnly()) + Attrs.push_back(Attribute::get(Ctx, Attribute::InaccessibleMemOnly)); + else if (isAssumedArgMemOnly()) + Attrs.push_back(Attribute::get(Ctx, Attribute::ArgMemOnly)); + else if (isAssumedInaccessibleOrArgMemOnly()) + Attrs.push_back( + Attribute::get(Ctx, Attribute::InaccessibleMemOrArgMemOnly)); + assert(Attrs.size() <= 2); } /// An AA to represent the memory behavior function attributes. @@ -1499,8 +1578,118 @@ virtual ManifestPosition getManifestPosition() const override { return MP_FUNCTION; } + +protected: + /// Encoding of different locations that could be accessed by a memory access. + enum MemoryLocationKind { + MLK_NONE = 0, + MLK_CONST_OR_LOCAL = 1 << 0, + MLK_GLOBAL = 1 << 1, + MLK_ARGUMENT = 1 << 2, + MLK_INACCESSIBLE = 1 << 3, + MLK_UNKNOWN = + MLK_CONST_OR_LOCAL | MLK_GLOBAL | MLK_ARGUMENT | MLK_INACCESSIBLE, + }; + + /// Return the kind(s) of location that may be accessed by \p I. + MemoryLocationKind + categorizeAccessedLocations(Attributor &A, const Instruction &I, + const AAMemoryBehavior *MemBehaviorAA); + + /// Return the kind(s) of location that may be represented by \p V. + MemoryLocationKind + categorizeRepresentedLocations(Attributor &A, const Value &V, + SmallPtrSetImpl &Visited); }; +AAMemoryBehaviorFunction::MemoryLocationKind +AAMemoryBehaviorFunction::categorizeRepresentedLocations( + Attributor &A, const Value &V, SmallPtrSetImpl &Visited) { + if (!Visited.insert(&V).second) + return MLK_NONE; + + const Value *Ptr, *PtrOrigin = V.stripInBoundsConstantOffsets(); + do { + Ptr = PtrOrigin; + if (const BitCastInst *BCI = dyn_cast(Ptr)) + PtrOrigin = BCI->getOperand(0); + else if (const GetElementPtrInst *GEP = dyn_cast(Ptr)) + PtrOrigin = GEP->getPointerOperand(); + else if (const SelectInst *SI = dyn_cast(Ptr)) { + return (MemoryLocationKind)( + categorizeRepresentedLocations(A, *SI->getTrueValue(), Visited) | + categorizeRepresentedLocations(A, *SI->getFalseValue(), Visited)); + } else if (const PHINode *PHI = dyn_cast(Ptr)) { + MemoryLocationKind MLK = MLK_NONE; + for (const Value *PHIOp : PHI->operands()) + MLK = (MemoryLocationKind)( + MLK | categorizeRepresentedLocations(A, *PHIOp, Visited)); + return MLK; + } + } while (PtrOrigin != Ptr); + + if (isa(Ptr)) + return MLK_NONE; + if (isa(Ptr)) + return MLK_ARGUMENT; + if (isa(Ptr)) + return MLK_GLOBAL; + if (AAR.pointsToConstantMemory(Ptr, /*OrLocal=*/true)) + return MLK_CONST_OR_LOCAL; + + return MLK_UNKNOWN; +} + +AAMemoryBehaviorFunction::MemoryLocationKind +AAMemoryBehaviorFunction::categorizeAccessedLocations( + Attributor &A, const Instruction &I, + const AAMemoryBehavior *MemBehaviorAA) { + SmallPtrSet VisitedValues; + + auto CollectArgumentLocations = [&](const ImmutableCallSite &ICS) { + MemoryLocationKind MLK = MLK_NONE; + unsigned ArgNo = 0; + for (Value *ArgOp : ICS.args()) { + // If the argument is readnone we ignore it. + auto *MemBehaviorAA = + A.getAAFor(*this, *ICS.getCalledValue(), ArgNo++); + if (MemBehaviorAA && MemBehaviorAA->getState().isValidState() && + MemBehaviorAA->isAssumedReadNone()) + continue; + + MLK = (MemoryLocationKind)( + MLK | categorizeRepresentedLocations(A, *ArgOp, VisitedValues)); + } + return MLK; + }; + + // We can try to lookup another in-flight abstract attribute for the + // instruction in question. + if (MemBehaviorAA && MemBehaviorAA->getState().isValidState()) { + if (MemBehaviorAA->isAssumedReadNone()) + return MLK_NONE; + if (MemBehaviorAA->isAssumedInaccessibleMemOnly()) + return MLK_INACCESSIBLE; + + ImmutableCallSite ICS(&I); + if (ICS && MemBehaviorAA->isAssumedArgMemOnly()) + return CollectArgumentLocations(ICS); + if (ICS && MemBehaviorAA->isAssumedInaccessibleOrArgMemOnly()) + return (MemoryLocationKind)(MLK_INACCESSIBLE | + CollectArgumentLocations(ICS)); + } + + // TODO: Memory intrinsics + Optional Loc = MemoryLocation::getOrNone(&I); + if (!Loc.hasValue()) + return MLK_UNKNOWN; + + if (AAR.pointsToConstantMemory(Loc.getValue(), /*OrLocal=*/true)) + return MLK_CONST_OR_LOCAL; + + return categorizeRepresentedLocations(A, *Loc->Ptr, VisitedValues); +} + ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) { // The current assumed state used to determine a change. @@ -1521,6 +1710,22 @@ continue; } + // If the instruction does not have an own memory behavior state, we + // categorize all potentially accessed locations and the potential + // side-effects explicitly. + MemoryLocationKind MLK = categorizeAccessedLocations(A, *I, MemBehaviorAA); + if (MLK == MLK_NONE || MLK == MLK_CONST_OR_LOCAL) + continue; + + // Remove location modifiers if necessary. + if (MLK & MLK_GLOBAL) + removeAssumedBits(ONLY_ARGUMENT_MEM | ONLY_INACCESSIBLE_MEM | + ONLY_INACCESSIBLE_OR_ARGUMENT_MEM); + else if (MLK & MLK_ARGUMENT) + removeAssumedBits(ONLY_INACCESSIBLE_MEM); + else if (MLK & MLK_INACCESSIBLE) + removeAssumedBits(ONLY_ARGUMENT_MEM); + // Remove access kind modifiers if necessary. if (I->mayReadFromMemory()) removeAssumedBits(NO_READS); @@ -1546,11 +1751,9 @@ F.removeFnAttr(Attribute::ReadNone); F.removeFnAttr(Attribute::ReadOnly); F.removeFnAttr(Attribute::WriteOnly); - if (isAssumedReadNone()) { - F.removeFnAttr(Attribute::ArgMemOnly); - F.removeFnAttr(Attribute::InaccessibleMemOnly); - F.removeFnAttr(Attribute::InaccessibleMemOrArgMemOnly); - } + F.removeFnAttr(Attribute::ArgMemOnly); + F.removeFnAttr(Attribute::InaccessibleMemOnly); + F.removeFnAttr(Attribute::InaccessibleMemOrArgMemOnly); return AbstractAttribute::manifest(A); } @@ -1565,6 +1768,11 @@ void initialize(Attributor &A) override { Argument &Arg = cast(getAnchoredValue()); + // Reset the memory location flags inherited from the AAMemoryBehaviorImpl + // class. The best for an argument is the combination of no-reads and + // no-writes => readnone. + removeAssumedBits(NO_LOCATIONS); + if (Arg.getNumUses() == 0) addKnownBits(NO_ACCESSES); else