diff --git a/openmp/libomptarget/src/device.h b/openmp/libomptarget/src/device.h --- a/openmp/libomptarget/src/device.h +++ b/openmp/libomptarget/src/device.h @@ -82,6 +82,8 @@ /// movement has been issued. This mutex *must* be locked right before /// releasing the mapping table lock. std::mutex UpdateMtx; + /// Pointer to the event corresponding to the data update of this map. + void *Event = nullptr; }; // When HostDataToTargetTy is used by std::set, std::set::iterator is const // use unique_ptr to make States mutable. @@ -115,6 +117,12 @@ /// Get the hold reference count. uint64_t getHoldRefCount() const { return States->HoldRefCount; } + /// Get the event bound to this data map. + void *getEvent() const { return States->Event; } + + /// Set the event bound to this data map. + void setEvent(void *Event) const { States->Event = Event; } + /// Reset the specified reference count unless it's infinity. Reset to 1 /// (even if currently 0) so it can be followed by a decrement. void resetRefCount(bool UseHoldRefCount) const { diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -91,6 +91,9 @@ "count\n"); } else if (search->isDynRefCountInf()) { DP("Association found, removing it\n"); + void *Event = search->getEvent(); + if (Event) + destroyEvent(Event); HostDataToTargetMap.erase(search); DataMapMtx.unlock(); return OFFLOAD_SUCCESS; @@ -264,20 +267,70 @@ DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); - - // Unlock the entry immediately after the data movement is issued. - Entry->unlock(); - if (Ret != OFFLOAD_SUCCESS) { + Entry->unlock(); REPORT("Copying data to device failed.\n"); // We will also return nullptr if the data movement fails because that // pointer points to a corrupted memory region so it doesn't make any // sense to continue to use it. TargetPointer = nullptr; } + + void *Event = nullptr; + Ret = createEvent(&Event); + if (Ret != OFFLOAD_SUCCESS) { + Entry->unlock(); + REPORT("Failed to create event\n"); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + + // We cannot assume the event should not be nullptr because we don't + // know if the target support event. But if a target doesn't, + // recordEvent should always return success. + Ret = recordEvent(Event, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + Entry->unlock(); + REPORT("Failed to set dependence on event " DPxMOD "\n", DPxPTR(Event)); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + void *OldEvent = Entry->getEvent(); + Entry->setEvent(Event); + // We're done with the entry. Release the entry. + Entry->unlock(); + // If there is an event attached, destroy it. + if (OldEvent) { + Ret = destroyEvent(OldEvent); + if (Ret != OFFLOAD_SUCCESS) + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } } else { // Release the mapping table lock directly. DataMapMtx.unlock(); + // If not a host pointer, we need to wait for the event if it exists. + if (!IsHostPtr) { + Entry->lock(); + void *Event = Entry->getEvent(); + if (Event) { + int Ret = waitEvent(Event, AsyncInfo); + Entry->unlock(); + if (Ret != OFFLOAD_SUCCESS) { + // If it fails to wait for the event, we need to return nullptr in + // case of any data race. + REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); + return {{false /* IsNewEntry */, false /* IsHostPointer */}, + {} /* MapTableEntry */, + nullptr /* TargetPointer */}; + } + } else { + Entry->unlock(); + } + } } return {{IsNew, IsHostPtr}, Entry, TargetPointer}; @@ -365,7 +418,7 @@ int DeviceTy::deallocTgtPtr(void *HstPtrBegin, int64_t Size, bool HasHoldModifier) { // Check if the pointer is contained in any sub-nodes. - int rc; + int Ret = OFFLOAD_SUCCESS; DataMapMtx.lock(); LookupResult lr = lookupMapping(HstPtrBegin, Size); if (lr.Flags.IsContained || lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter) { @@ -380,18 +433,22 @@ DPxPTR(HT.HstPtrBegin), DPxPTR(HT.TgtPtrBegin), Size, (HT.HstPtrName) ? getNameFromMapping(HT.HstPtrName).c_str() : "unknown"); + void *Event = lr.Entry->getEvent(); HostDataToTargetMap.erase(lr.Entry); + if (Event && destroyEvent(Event) != OFFLOAD_SUCCESS) { + REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event)); + Ret = OFFLOAD_FAIL; + } } - rc = OFFLOAD_SUCCESS; } else { REPORT("Section to delete (hst addr " DPxMOD ") does not exist in the" " allocated memory\n", DPxPTR(HstPtrBegin)); - rc = OFFLOAD_FAIL; + Ret = OFFLOAD_FAIL; } DataMapMtx.unlock(); - return rc; + return Ret; } /// Init device, should not be called directly. diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -581,14 +581,36 @@ void *&TgtPtrBase = AsyncInfo.getVoidPtrLocation(); TgtPtrBase = ExpectedTgtPtrBase; - int rt = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, - sizeof(void *), AsyncInfo); - Pointer_TPR.MapTableEntry->unlock(); - - if (rt != OFFLOAD_SUCCESS) { + int Ret = Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, + sizeof(void *), AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + Pointer_TPR.MapTableEntry->unlock(); REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } + void *Event = nullptr; + if (Device.createEvent(&Event) != OFFLOAD_SUCCESS) { + Pointer_TPR.MapTableEntry->unlock(); + REPORT("Failed to create event.\n"); + return OFFLOAD_FAIL; + } + // We cannot assume the event should not be nullptr because we don't + // know if the target support event. But if a target doesn't, + // recordEvent should always return success. + Ret = Device.recordEvent(Event, AsyncInfo); + if (Ret != OFFLOAD_SUCCESS) { + Pointer_TPR.MapTableEntry->unlock(); + REPORT("Failed to set dependence on event " DPxMOD "\n", + DPxPTR(Event)); + return OFFLOAD_FAIL; + } + // Exchange the old event with new created event + void *OldEvent = Pointer_TPR.MapTableEntry->getEvent(); + Pointer_TPR.MapTableEntry->setEvent(Event); + Pointer_TPR.MapTableEntry->unlock(); + // If the old event is not null, we need to destroy it. + if (OldEvent) + Device.destroyEvent(OldEvent); } else Device.ShadowMtx.unlock(); }