Please use GitHub pull requests for new patches. Avoid migrating existing patches. Phabricator shutdown timeline

Differential D142514 Diff 493102 openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h

Changeset View

Standalone View

openmp/libomptarget/plugins-nextgen/common/PluginInterface/PluginInterface.h

Show First 20 Lines • Show All 263 Lines • ▼ Show 20 Lines	struct EntryTy {
/// The pointer that devices' driver should use to transfer data from/to the		/// The pointer that devices' driver should use to transfer data from/to the
/// pinned allocation. In most plugins, this pointer will be the same as the		/// pinned allocation. In most plugins, this pointer will be the same as the
/// host pointer above.		/// host pointer above.
void *DevAccessiblePtr;		void *DevAccessiblePtr;

/// The size of the pinned allocation.		/// The size of the pinned allocation.
size_t Size;		size_t Size;

		/// Indicate whether the allocation was locked from outside the plugin, for
		/// instance, from the application. The externally locked allocations are
		/// not unlocked by the plugin when unregistering the last user.
		bool ExternallyLocked;

/// The number of references to the pinned allocation. The allocation should		/// The number of references to the pinned allocation. The allocation should
/// remain pinned and registered to the map until the number of references		/// remain pinned and registered to the map until the number of references
/// becomes zero.		/// becomes zero.
mutable size_t References;		mutable size_t References;

/// Create an entry with the host and device acessible pointers, and the		/// Create an entry with the host and device acessible pointers, the buffer
/// buffer size.		/// size, and a boolean indicating whether the buffer was locked externally.
EntryTy(void HstPtr, void DevAccessiblePtr, size_t Size)		EntryTy(void HstPtr, void DevAccessiblePtr, size_t Size,
		bool ExternallyLocked)
: HstPtr(HstPtr), DevAccessiblePtr(DevAccessiblePtr), Size(Size),		: HstPtr(HstPtr), DevAccessiblePtr(DevAccessiblePtr), Size(Size),
References(1) {}		ExternallyLocked(ExternallyLocked), References(1) {}

/// Utility constructor used for std::set searches.		/// Utility constructor used for std::set searches.
EntryTy(void *HstPtr)		EntryTy(void *HstPtr)
: HstPtr(HstPtr), DevAccessiblePtr(nullptr), Size(0), References(0) {}		: HstPtr(HstPtr), DevAccessiblePtr(nullptr), Size(0),
		ExternallyLocked(false), References(0) {}
};		};

/// Comparator of mep entries. Use the host pointer to enforce an order		/// Comparator of mep entries. Use the host pointer to enforce an order
/// between entries.		/// between entries.
struct EntryCmpTy {		struct EntryCmpTy {
bool operator()(const EntryTy &Left, const EntryTy &Right) const {		bool operator()(const EntryTy &Left, const EntryTy &Right) const {
return Left.HstPtr < Right.HstPtr;		return Left.HstPtr < Right.HstPtr;
}		}
};		};

typedef std::set<EntryTy, EntryCmpTy> PinnedAllocSetTy;		typedef std::set<EntryTy, EntryCmpTy> PinnedAllocSetTy;

/// The map of host pinned allocations.		/// The map of host pinned allocations.
PinnedAllocSetTy Allocs;		PinnedAllocSetTy Allocs;

/// The mutex to protect accesses to the map.		/// The mutex to protect accesses to the map.
mutable std::shared_mutex Mutex;		mutable std::shared_mutex Mutex;

/// Reference to the corresponding device.		/// Reference to the corresponding device.
GenericDeviceTy &Device;		GenericDeviceTy &Device;

/// Find an allocation that intersects with \p Buffer pointer. Assume		/// Envar that indicates whether mapped host bufffers should be locked
/// the map's mutex is acquired.		/// automatically.
PinnedAllocSetTy::iterator findIntersecting(const void *Buffer) const {		BoolEnvar OMPX_LockMappedBuffers;

		/// Find an allocation that intersects with \p HstPtr pointer. Assume the
		/// map's mutex is acquired.
		const EntryTy findIntersecting(const void HstPtr) const {
if (Allocs.empty())		if (Allocs.empty())
return Allocs.end();		return nullptr;

// Search the first allocation with starting address that is not less than		// Search the first allocation with starting address that is not less than
// the buffer address.		// the buffer address.
auto It = Allocs.lower_bound({const_cast<void *>(Buffer)});		auto It = Allocs.lower_bound({const_cast<void *>(HstPtr)});

// Direct match of starting addresses.		// Direct match of starting addresses.
if (It != Allocs.end() && It->HstPtr == Buffer)		if (It != Allocs.end() && It->HstPtr == HstPtr)
return It;		return &(*It);

// Not direct match but may be a previous pinned allocation in the map which		// Not direct match but may be a previous pinned allocation in the map which
// contains the buffer. Return false if there is no such a previous		// contains the buffer. Return false if there is no such a previous
// allocation.		// allocation.
if (It == Allocs.begin())		if (It == Allocs.begin())
return Allocs.end();		return nullptr;

// Move to the previous pinned allocation.		// Move to the previous pinned allocation.
--It;		--It;

// The buffer is not contained in the pinned allocation.		// The buffer is not contained in the pinned allocation.
if (advanceVoidPtr(It->HstPtr, It->Size) > Buffer)		if (advanceVoidPtr(It->HstPtr, It->Size) > HstPtr)
return It;		return &(*It);

// None found.		// None found.
return Allocs.end();		return nullptr;
		}

		/// Insert an entry to the map representing a locked buffer. The number of
		/// references is set to one.
		Error insertEntry(void HstPtr, void DevAccessiblePtr, size_t Size,
		bool ExternallyLocked = false);

		/// Erase an existing entry from the map.
		Error eraseEntry(const EntryTy &Entry);

		/// Register a new user into an entry that represents a locked buffer. Check
		/// also that the registered buffer with \p HstPtr address and \p Size is
		/// actually contained into the entry.
		Error registerEntryUse(const EntryTy &Entry, void *HstPtr, size_t Size);

		/// Unregister a user from the entry and return whether it is the last user.
		/// If it is the last user, the entry will have to be removed from the map
		/// and unlock the entry's host buffer (if necessary).
		Expected<bool> unregisterEntryUse(const EntryTy &Entry);

		/// Indicate whether the first range A fully contains the second range B.
		static bool contains(void PtrA, size_t SizeA, void PtrB, size_t SizeB) {
		void *EndA = advanceVoidPtr(PtrA, SizeA);
		void *EndB = advanceVoidPtr(PtrB, SizeB);
		return (PtrB >= PtrA && EndB <= EndA);
		}

		/// Indicate whether the first range A intersects with the second range B.
		static bool intersects(void PtrA, size_t SizeA, void PtrB, size_t SizeB) {
		void *EndA = advanceVoidPtr(PtrA, SizeA);
		void *EndB = advanceVoidPtr(PtrB, SizeB);
		return (PtrA < EndB && PtrB < EndA);
}		}

public:		public:
/// Create the map of pinned allocations corresponding to a specific device.		/// Create the map of pinned allocations corresponding to a specific device.
PinnedAllocationMapTy(GenericDeviceTy &Device) : Device(Device) {}		PinnedAllocationMapTy(GenericDeviceTy &Device)
		: Device(Device),
		OMPX_LockMappedBuffers("LIBOMPTARGET_LOCK_MAPPED_HOST_BUFFERS", false) {
		}

/// Register a host buffer that was recently locked. None of the already		/// Register a buffer that was recently allocated as a locked host buffer.
/// registered pinned allocations should intersect with this new one. The		/// None of the already registered pinned allocations should intersect with
/// registration requires the host pointer in \p HstPtr, the pointer that the		/// this new one. The registration requires the host pointer in \p HstPtr,
/// devices should use when transferring data from/to the allocation in		/// the device accessible pointer in \p DevAccessiblePtr, and the size of the
/// \p DevAccessiblePtr, and the size of the allocation in \p Size. Notice		/// allocation in \p Size. The allocation must be unregistered using the
/// that some plugins may use the same pointer for the \p HstPtr and
/// \p DevAccessiblePtr. The allocation must be unregistered using the
/// unregisterHostBuffer function.		/// unregisterHostBuffer function.
Error registerHostBuffer(void HstPtr, void DevAccessiblePtr, size_t Size);		Error registerHostBuffer(void HstPtr, void DevAccessiblePtr, size_t Size);

/// Unregister a host pinned allocation passing the host pointer which was		/// Unregister a host pinned allocation passing the host pointer which was
/// previously registered using the registerHostBuffer function. When calling		/// previously registered using the registerHostBuffer function. When calling
/// this function, the pinned allocation cannot have any other user.		/// this function, the pinned allocation cannot have any other user and will
		/// not be unlocked by this function.
Error unregisterHostBuffer(void *HstPtr);		Error unregisterHostBuffer(void *HstPtr);

/// Lock the host buffer at \p HstPtr or register a new user if it intersects		/// Lock the host buffer at \p HstPtr or register a new user if it intersects
/// with an already existing one. A partial overlapping with extension is not		/// with an already existing one. A partial overlapping with extension is not
/// allowed. The function returns the device accessible pointer of the pinned		/// allowed. The function returns the device accessible pointer of the pinned
/// buffer. The buffer must be unlocked using the unlockHostBuffer function.		/// buffer. The buffer must be unlocked using the unlockHostBuffer function.
Expected<void > lockHostBuffer(void HstPtr, size_t Size);		Expected<void > lockHostBuffer(void HstPtr, size_t Size);

/// Unlock the host buffer at \p HstPtr or unregister a user if other users		/// Unlock the host buffer at \p HstPtr or unregister a user if other users
/// are still using the pinned allocation. If this was the last user, the		/// are still using the pinned allocation. If this was the last user, the
/// pinned allocation is removed from the map and the memory is unlocked.		/// pinned allocation is removed from the map and the memory is unlocked.
Error unlockHostBuffer(void *HstPtr);		Error unlockHostBuffer(void *HstPtr);

		/// Lock or register a host buffer that was recently mapped by libomptarget.
		/// This behavior is applied if LIBOMPTARGET_LOCK_MAPPED_HOST_BUFFERS is
		/// enabled. Even if not enabled, externally locked buffers are registered
		/// in order to optimize their transfers.
		Error lockMappedHostBuffer(void *HstPtr, size_t Size);

		/// Unlock or unregister a host buffer that was unmapped by libomptarget.
		Error unlockUnmappedHostBuffer(void *HstPtr);

/// Return the device accessible pointer associated to the host pinned		/// Return the device accessible pointer associated to the host pinned
/// allocation which the \p HstPtr belongs, if any. Return null in case the		/// allocation which the \p HstPtr belongs, if any. Return null in case the
/// \p HstPtr does not belong to any host pinned allocation. The device		/// \p HstPtr does not belong to any host pinned allocation. The device
/// accessible pointer is the one that devices should use for data transfers		/// accessible pointer is the one that devices should use for data transfers
/// that involve a host pinned buffer.		/// that involve a host pinned buffer.
void getDeviceAccessiblePtrFromPinnedBuffer(const void HstPtr) const {		void getDeviceAccessiblePtrFromPinnedBuffer(const void HstPtr) const {
std::shared_lock<std::shared_mutex> Lock(Mutex);		std::shared_lock<std::shared_mutex> Lock(Mutex);

// Find the intersecting allocation if any.		// Find the intersecting allocation if any.
auto It = findIntersecting(HstPtr);		const EntryTy *Entry = findIntersecting(HstPtr);
if (It == Allocs.end())		if (!Entry)
return nullptr;		return nullptr;

const EntryTy &Entry = *It;		return advanceVoidPtr(Entry->DevAccessiblePtr,
return advanceVoidPtr(Entry.DevAccessiblePtr,		getPtrDiff(HstPtr, Entry->HstPtr));
getPtrDiff(HstPtr, Entry.HstPtr));
}		}

/// Check whether a buffer belongs to a registered host pinned allocation.		/// Check whether a buffer belongs to a registered host pinned allocation.
bool isHostPinnedBuffer(const void *HstPtr) const {		bool isHostPinnedBuffer(const void *HstPtr) const {
std::shared_lock<std::shared_mutex> Lock(Mutex);		std::shared_lock<std::shared_mutex> Lock(Mutex);

// Return whether there is an intersecting allocation.		// Return whether there is an intersecting allocation.
return (findIntersecting(const_cast<void *>(HstPtr)) != Allocs.end());		return (findIntersecting(const_cast<void *>(HstPtr)) != nullptr);
}		}
};		};

/// Class implementing common functionalities of offload devices. Each plugin		/// Class implementing common functionalities of offload devices. Each plugin
/// should define the specific device class, derive from this generic one, and		/// should define the specific device class, derive from this generic one, and
/// implement the necessary virtual function members.		/// implement the necessary virtual function members.
struct GenericDeviceTy : public DeviceAllocatorTy {		struct GenericDeviceTy : public DeviceAllocatorTy {
/// Construct a device with its device id within the plugin, the number of		/// Construct a device with its device id within the plugin, the number of
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	struct GenericDeviceTy : public DeviceAllocatorTy {

/// Pin host memory to optimize transfers and return the device accessible		/// Pin host memory to optimize transfers and return the device accessible
/// pointer that devices should use for memory transfers involving the host		/// pointer that devices should use for memory transfers involving the host
/// pinned allocation.		/// pinned allocation.
Expected<void > dataLock(void HstPtr, int64_t Size) {		Expected<void > dataLock(void HstPtr, int64_t Size) {
return PinnedAllocs.lockHostBuffer(HstPtr, Size);		return PinnedAllocs.lockHostBuffer(HstPtr, Size);
}		}

virtual Expected<void > dataLockImpl(void HstPtr, int64_t Size) = 0;

/// Unpin a host memory buffer that was previously pinned.		/// Unpin a host memory buffer that was previously pinned.
Error dataUnlock(void *HstPtr) {		Error dataUnlock(void *HstPtr) {
return PinnedAllocs.unlockHostBuffer(HstPtr);		return PinnedAllocs.unlockHostBuffer(HstPtr);
}		}

		/// Lock the host buffer \p HstPtr with \p Size bytes with the vendor-specific
		/// API and return the device accessible pointer.
		virtual Expected<void > dataLockImpl(void HstPtr, int64_t Size) = 0;

		/// Unlock a previously locked host buffer starting at \p HstPtr.
virtual Error dataUnlockImpl(void *HstPtr) = 0;		virtual Error dataUnlockImpl(void *HstPtr) = 0;

		/// Mark the host buffer with address \p HstPtr and \p Size bytes as a mapped
		/// buffer. This means that libomptarget created a new mapping of that host
		/// buffer (e.g., because a user OpenMP target map) and the buffer may be used
		/// as source/destination of memory transfers. We can use this information to
		/// lock the host buffer and optimize its memory transfers.
		Error notifyDataMapped(void *HstPtr, int64_t Size) {
		return PinnedAllocs.lockMappedHostBuffer(HstPtr, Size);
		}

		jdoerfertUnsubmitted Done Reply Inline Actions Docs jdoerfert: Docs
		/// Mark the host buffer with address \p HstPtr as unmapped. This means that
		/// libomptarget removed an existing mapping. If the plugin locked the buffer
		/// in notifyDataMapped, this function should unlock it.
		Error notifyDataUnmapped(void *HstPtr) {
		return PinnedAllocs.unlockUnmappedHostBuffer(HstPtr);
		}

		/// Check whether the host buffer with address \p HstPtr is pinned by the
		/// underlying vendor-specific runtime (if any). Retrieve the host pointer,
		/// the device accessible pointer and the size of the original pinned buffer.
		virtual Expected<bool> isPinnedPtrImpl(void HstPtr, void &BaseHstPtr,
		void *&BaseDevAccessiblePtr,
		size_t &BaseSize) const = 0;

/// Submit data to the device (host to device transfer).		/// Submit data to the device (host to device transfer).
Error dataSubmit(void TgtPtr, const void HstPtr, int64_t Size,		Error dataSubmit(void TgtPtr, const void HstPtr, int64_t Size,
__tgt_async_info *AsyncInfo);		__tgt_async_info *AsyncInfo);
virtual Error dataSubmitImpl(void TgtPtr, const void HstPtr, int64_t Size,		virtual Error dataSubmitImpl(void TgtPtr, const void HstPtr, int64_t Size,
AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;		AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;

/// Retrieve data from the device (device to host transfer).		/// Retrieve data from the device (device to host transfer).
Error dataRetrieve(void HstPtr, const void TgtPtr, int64_t Size,		Error dataRetrieve(void HstPtr, const void TgtPtr, int64_t Size,
▲ Show 20 Lines • Show All 531 Lines • Show Last 20 Lines