diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h --- a/openmp/libomptarget/include/device.h +++ b/openmp/libomptarget/include/device.h @@ -275,19 +275,46 @@ /// This struct will be returned by \p DeviceTy::getTargetPointer which provides /// more data than just a target pointer. -struct TargetPointerResultTy { - struct { - /// If the map table entry is just created +class TargetPointerResultTy { + struct FlagTy { + /// Flag indicating that the map table entry was just created. unsigned IsNewEntry : 1; - /// If the pointer is actually a host pointer (when unified memory enabled) + /// Flag indicating that the pointer is actually a host pointer, used when + /// unified memory enabled. unsigned IsHostPointer : 1; - } Flags = {0, 0}; + /// Flag indicating that this was the last user of the entry and the ref + /// count is now 0. + unsigned IsLast : 1; + } Flags = {0, 0, 0}; /// The corresponding map table entry which is stable. HostDataToTargetTy *Entry = nullptr; /// The corresponding target pointer void *TargetPointer = nullptr; + +public: + TargetPointerResultTy() {} + + TargetPointerResultTy(FlagTy Flags, HostDataToTargetTy *Entry, + void *TargetPointer) + : Flags(Flags), Entry(Entry), TargetPointer(TargetPointer) { + } + + bool isHostPtr() const { return Flags.IsHostPointer; } + void setIsHostPtr(bool IHP) { Flags.IsHostPointer = IHP; } + + bool isNew() const { return Flags.IsNewEntry; } + void setIsNew(bool IN) { Flags.IsNewEntry = IN; } + + bool isLast() const { return Flags.IsLast; } + void setIsLast(bool IL) { Flags.IsLast = IL; } + + void *getTargetPointer() const { return TargetPointer; } + void setTargetPointer(void *TP) { TargetPointer = TP; } + + HostDataToTargetTy *getEntry() const { return Entry; } + void setEntry(HostDataToTargetTy *HDTTT) { Entry = HDTTT; } }; /// Map for shadow pointers @@ -364,11 +391,12 @@ /// - The user tried to do an illegal mapping; /// - Data transfer issue fails. TargetPointerResultTy - getTargetPointer(void *HstPtrBegin, void *HstPtrBase, int64_t Size, - map_var_info_t HstPtrName, bool HasFlagTo, - bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, - bool HasCloseModifier, bool HasPresentModifier, - bool HasHoldModifier, AsyncInfoTy &AsyncInfo); + getTargetPointer(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, + void *HstPtrBase, int64_t Size, map_var_info_t HstPtrName, + bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, + bool UpdateRefCount, bool HasCloseModifier, + bool HasPresentModifier, bool HasHoldModifier, + AsyncInfoTy &AsyncInfo); /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor /// ensures exclusive access to the HDTT map. @@ -376,8 +404,8 @@ int64_t Size); TargetPointerResultTy getTgtPtrBegin(void *HstPtrBegin, int64_t Size, - bool &IsLast, bool UpdateRefCount, - bool UseHoldRefCount, bool &IsHostPtr, + bool UpdateRefCount, + bool UseHoldRefCount, bool MustContain = false, bool ForceDelete = false); diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp --- a/openmp/libomptarget/src/api.cpp +++ b/openmp/libomptarget/src/api.cpp @@ -106,19 +106,16 @@ } DeviceTy &Device = *PM->Devices[device_num]; - bool IsLast; // not used - bool IsHostPtr; - TargetPointerResultTy TPR = - Device.getTgtPtrBegin(const_cast(ptr), 0, IsLast, - /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false, IsHostPtr); - int rc = (TPR.TargetPointer != NULL); + TargetPointerResultTy TPR = Device.getTgtPtrBegin(const_cast(ptr), 0, + /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); + int rc = (TPR.getTargetPointer() != NULL); // Under unified memory the host pointer can be returned by the // getTgtPtrBegin() function which means that there is no device // corresponding point for ptr. This function should return false // in that situation. if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) - rc = !IsHostPtr; + rc = !TPR.isHostPtr(); DP("Call to omp_target_is_present returns %d\n", rc); return rc; } diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -209,18 +209,18 @@ } TargetPointerResultTy DeviceTy::getTargetPointer( - void *HstPtrBegin, void *HstPtrBase, int64_t Size, - map_var_info_t HstPtrName, bool HasFlagTo, bool HasFlagAlways, + HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase, + int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount, bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier, AsyncInfoTy &AsyncInfo) { - HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); - - void *TargetPointer = nullptr; - bool IsHostPtr = false; - bool IsNew = false; + TargetPointerResultTy TPR; LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size); - auto *Entry = LR.Entry; + + TPR.setEntry(LR.Entry); + + // Not that the entry can be null. + HostDataToTargetTy *Entry = TPR.getEntry(); // Check if the pointer is contained. // If a variable is mapped to the device manually by the user - which would @@ -238,7 +238,7 @@ RefCountAction = " (incremented)"; } else { // It might have been allocated with the parent, but it's still new. - IsNew = HT.getTotalRefCount() == 1; + TPR.setIsNew(HT.getTotalRefCount() == 1); RefCountAction = " (update suppressed)"; } const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction; @@ -251,7 +251,7 @@ Size, HT.dynRefCountToStr().c_str(), DynRefCountAction, HT.holdRefCountToStr().c_str(), HoldRefCountAction, (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); - TargetPointer = (void *)Ptr; + TPR.setTargetPointer((void *)Ptr); } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) { // Explicit extension of mapped data - not allowed. MESSAGE("explicit extension not allowed: host address specified is " DPxMOD @@ -276,8 +276,8 @@ DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " "memory\n", DPxPTR((uintptr_t)HstPtrBegin), Size); - IsHostPtr = true; - TargetPointer = HstPtrBegin; + TPR.setIsHostPtr(true); + TPR.setTargetPointer(HstPtrBegin); } } else if (HasPresentModifier) { DP("Mapping required by 'present' map type modifier does not exist for " @@ -288,7 +288,7 @@ DPxPTR(HstPtrBegin), Size); } else if (Size) { // If it is not contained and Size > 0, we should create a new entry for it. - IsNew = true; + TPR.setIsNew(true); uintptr_t Ptr = (uintptr_t)allocData(Size, HstPtrBegin); Entry = HDTTMap ->emplace(new HostDataToTargetTy( @@ -296,6 +296,8 @@ (uintptr_t)HstPtrBegin + Size, Ptr, HasHoldModifier, HstPtrName)) .first->HDTT; + TPR.setEntry(Entry); + INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID, "Creating new map entry with HstPtrBase= " DPxMOD ", HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", Size=%ld, " @@ -303,40 +305,34 @@ DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(Ptr), Size, Entry->dynRefCountToStr().c_str(), Entry->holdRefCountToStr().c_str(), (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown"); - TargetPointer = (void *)Ptr; + TPR.setTargetPointer((void *)Ptr); } // If the target pointer is valid, and we need to transfer data, issue the // data transfer. - if (TargetPointer && !IsHostPtr && HasFlagTo && (IsNew || HasFlagAlways)) { - // Lock the entry before releasing the mapping table lock such that another - // thread that could issue data movement will get the right result. - std::lock_guard LG(*Entry); - // Release the mapping table lock right after the entry is locked. - HDTTMap.destroy(); - + if (TPR.getTargetPointer() && !TPR.isHostPtr() && HasFlagTo && + (TPR.isNew() || HasFlagAlways)) { DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size, - DPxPTR(HstPtrBegin), DPxPTR(TargetPointer)); + DPxPTR(HstPtrBegin), DPxPTR(TPR.getTargetPointer())); - int Ret = submitData(TargetPointer, HstPtrBegin, Size, AsyncInfo); + int Ret = submitData(TPR.getTargetPointer(), HstPtrBegin, Size, AsyncInfo); if (Ret != OFFLOAD_SUCCESS) { REPORT("Copying data to device failed.\n"); // We will also return nullptr if the data movement fails because that // pointer points to a corrupted memory region so it doesn't make any // sense to continue to use it. - TargetPointer = nullptr; - } else if (Entry->addEventIfNecessary(*this, AsyncInfo) != OFFLOAD_SUCCESS) + TPR.setTargetPointer(nullptr); + } else if (Entry->addEventIfNecessary(*this, AsyncInfo) != + OFFLOAD_SUCCESS) { return {{false /* IsNewEntry */, false /* IsHostPointer */}, nullptr /* Entry */, nullptr /* TargetPointer */}; + } } else { - // Release the mapping table lock directly. - HDTTMap.destroy(); // If not a host pointer and no present modifier, we need to wait for the // event if it exists. // Note: Entry might be nullptr because of zero length array section. - if (Entry && !IsHostPtr && !HasPresentModifier) { - std::lock_guard LG(*Entry); + if (Entry && !TPR.isHostPtr() && !HasPresentModifier) { void *Event = Entry->getEvent(); if (Event) { int Ret = waitEvent(Event, AsyncInfo); @@ -352,39 +348,41 @@ } } - return {{IsNew, IsHostPtr}, Entry, TargetPointer}; + return TPR; } // Used by targetDataBegin, targetDataEnd, targetDataUpdate and target. // Return the target pointer begin (where the data will be moved). // Decrement the reference counter if called from targetDataEnd. -TargetPointerResultTy -DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool &IsLast, - bool UpdateRefCount, bool UseHoldRefCount, - bool &IsHostPtr, bool MustContain, bool ForceDelete) { - HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); - - void *TargetPointer = NULL; - bool IsNew = false; - IsHostPtr = false; - IsLast = false; - LookupResult lr = lookupMapping(HDTTMap, HstPtrBegin, Size); +TargetPointerResultTy DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, + bool UpdateRefCount, + bool UseHoldRefCount, + bool MustContain, + bool ForceDelete) { + + TargetPointerResultTy TPR; + LookupResult LR; + { + HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor(); + LR = lookupMapping(HDTTMap, HstPtrBegin, Size); + TPR.setEntry(LR.Entry); + } - if (lr.Flags.IsContained || - (!MustContain && (lr.Flags.ExtendsBefore || lr.Flags.ExtendsAfter))) { - auto &HT = *lr.Entry; - IsLast = HT.decShouldRemove(UseHoldRefCount, ForceDelete); + if (LR.Flags.IsContained || + (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) { + auto &HT = *LR.Entry; + TPR.setIsLast(HT.decShouldRemove(UseHoldRefCount, ForceDelete)); if (ForceDelete) { HT.resetRefCount(UseHoldRefCount); - assert(IsLast == HT.decShouldRemove(UseHoldRefCount) && + assert(TPR.isLast() == HT.decShouldRemove(UseHoldRefCount) && "expected correct IsLast prediction for reset"); } const char *RefCountAction; if (!UpdateRefCount) { RefCountAction = " (update suppressed)"; - } else if (IsLast) { + } else if (TPR.isLast()) { // Mark the entry as to be deleted by this thread. Another thread might // reuse the entry and take "ownership" for the deletion while this thread // is waiting for data transfers. That is fine and the current thread will @@ -409,7 +407,7 @@ "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n", DPxPTR(HstPtrBegin), DPxPTR(tp), Size, HT.dynRefCountToStr().c_str(), DynRefCountAction, HT.holdRefCountToStr().c_str(), HoldRefCountAction); - TargetPointer = (void *)tp; + TPR.setTargetPointer((void *)tp); } else if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) { // If the value isn't found in the mapping and unified shared memory // is on then it means we have stumbled upon a value which we need to @@ -417,11 +415,11 @@ DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared " "memory\n", DPxPTR((uintptr_t)HstPtrBegin), Size); - IsHostPtr = true; - TargetPointer = HstPtrBegin; + TPR.setIsHostPtr(true); + TPR.setTargetPointer(HstPtrBegin); } - return {{IsNew, IsHostPtr}, lr.Entry, TargetPointer}; + return TPR; } // Return the target pointer begin (where the data will be moved). diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -466,9 +466,8 @@ } // Address of pointer on the host and device, respectively. - void *Pointer_HstPtrBegin, *PointerTgtPtrBegin; + void **PointerHstPtrBegin, **PointerTgtPtrBegin; TargetPointerResultTy Pointer_TPR; - bool IsHostPtr = false; bool IsImplicit = arg_types[i] & OMP_TGT_MAPTYPE_IMPLICIT; // Force the creation of a device side copy of the data when: // a close map modifier was associated with a map that contained a to. @@ -482,6 +481,9 @@ // may be considered a hack, we could revise the scheme in the future. bool UpdateRef = !(arg_types[i] & OMP_TGT_MAPTYPE_MEMBER_OF) && !(FromMapper && i == 0); + + DeviceTy::HDTTMapAccessorTy HDTTMap = + Device.HostDataToTargetMap.getExclusiveAccessor(); if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) { DP("Has a pointer entry: \n"); // Base is address of pointer. @@ -498,11 +500,11 @@ // PTR_AND_OBJ entry is handled below, and so the allocation might fail // when HasPresentModifier. Pointer_TPR = Device.getTargetPointer( - HstPtrBase, HstPtrBase, sizeof(void *), /*HstPtrName=*/nullptr, + HDTTMap, HstPtrBase, HstPtrBase, sizeof(void *), + /*HstPtrName=*/nullptr, /*HasFlagTo=*/false, /*HasFlagAlways=*/false, IsImplicit, UpdateRef, HasCloseModifier, HasPresentModifier, HasHoldModifier, AsyncInfo); - PointerTgtPtrBegin = Pointer_TPR.TargetPointer; - IsHostPtr = Pointer_TPR.Flags.IsHostPointer; + PointerTgtPtrBegin = (void **)Pointer_TPR.getTargetPointer(); if (!PointerTgtPtrBegin) { REPORT("Call to getTargetPointer returned null pointer (%s).\n", HasPresentModifier ? "'present' map type modifier" @@ -512,10 +514,10 @@ DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new" "\n", sizeof(void *), DPxPTR(PointerTgtPtrBegin), - (Pointer_TPR.Flags.IsNewEntry ? "" : " not")); - Pointer_HstPtrBegin = HstPtrBase; + (Pointer_TPR.isNew() ? "" : " not")); + PointerHstPtrBegin = (void **)HstPtrBase; // modify current entry. - HstPtrBase = *(void **)HstPtrBase; + HstPtrBase = *PointerHstPtrBegin; // No need to update pointee ref count for the first element of the // subelement that comes from mapper. UpdateRef = @@ -525,11 +527,11 @@ const bool HasFlagTo = arg_types[i] & OMP_TGT_MAPTYPE_TO; const bool HasFlagAlways = arg_types[i] & OMP_TGT_MAPTYPE_ALWAYS; auto TPR = Device.getTargetPointer( - HstPtrBegin, HstPtrBase, data_size, HstPtrName, HasFlagTo, + HDTTMap, HstPtrBegin, HstPtrBase, data_size, HstPtrName, HasFlagTo, HasFlagAlways, IsImplicit, UpdateRef, HasCloseModifier, HasPresentModifier, HasHoldModifier, AsyncInfo); - void *TgtPtrBegin = TPR.TargetPointer; - IsHostPtr = TPR.Flags.IsHostPointer; + + void *TgtPtrBegin = TPR.getTargetPointer(); // If data_size==0, then the argument could be a zero-length pointer to // NULL, so getOrAlloc() returning NULL is not an error. if (!TgtPtrBegin && (data_size || HasPresentModifier)) { @@ -540,7 +542,7 @@ } DP("There are %" PRId64 " bytes allocated at target address " DPxMOD " - is%s new\n", - data_size, DPxPTR(TgtPtrBegin), (TPR.Flags.IsNewEntry ? "" : " not")); + data_size, DPxPTR(TgtPtrBegin), (TPR.isNew() ? "" : " not")); if (arg_types[i] & OMP_TGT_MAPTYPE_RETURN_PARAM) { uintptr_t Delta = (uintptr_t)HstPtrBegin - (uintptr_t)HstPtrBase; @@ -549,7 +551,7 @@ args_base[i] = TgtPtrBase; } - if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ && !IsHostPtr) { + if (arg_types[i] & OMP_TGT_MAPTYPE_PTR_AND_OBJ && !TPR.isHostPtr()) { // Check whether we need to update the pointer on the device bool UpdateDevPtr = false; @@ -557,7 +559,7 @@ void *ExpectedTgtPtrBase = (void *)((uint64_t)TgtPtrBegin - Delta); Device.ShadowMtx.lock(); - auto Entry = Device.ShadowPtrMap.find(Pointer_HstPtrBegin); + auto Entry = Device.ShadowPtrMap.find(PointerHstPtrBegin); // If this pointer is not in the map we need to insert it. If the map // contains a stale entry, we need to update it (e.g. if the pointee was // deallocated and later on is reallocated at another device address). The @@ -567,19 +569,20 @@ // env/base_ptr_ref_count.c the PTR is a global "declare target" pointer, // so it stays in the map for the lifetime of the application. When the // OBJ is deallocated and later on allocated again (at a different device - // address), ShadowPtrMap still contains an entry for Pointer_HstPtrBegin + // address), ShadowPtrMap still contains an entry for PointerHstPtrBegin // which is stale, pointing to the old ExpectedTgtPtrBase of the OBJ. if (Entry == Device.ShadowPtrMap.end() || Entry->second.TgtPtrVal != ExpectedTgtPtrBase) { // create or update shadow pointers for this entry - Device.ShadowPtrMap[Pointer_HstPtrBegin] = { + Device.ShadowPtrMap[PointerHstPtrBegin] = { HstPtrBase, PointerTgtPtrBegin, ExpectedTgtPtrBase}; - Pointer_TPR.Entry->setMayContainAttachedPointers(); + Pointer_TPR.getEntry()->setMayContainAttachedPointers(); UpdateDevPtr = true; } if (UpdateDevPtr) { - std::lock_guard LG(*Pointer_TPR.Entry); + std::lock_guard LG( + *Pointer_TPR.getEntry()); Device.ShadowMtx.unlock(); DP("Update pointer (" DPxMOD ") -> [" DPxMOD "]\n", @@ -594,7 +597,7 @@ REPORT("Copying data to device failed.\n"); return OFFLOAD_FAIL; } - if (Pointer_TPR.Entry->addEventIfNecessary(Device, AsyncInfo) != + if (Pointer_TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) != OFFLOAD_SUCCESS) return OFFLOAD_FAIL; } else @@ -647,7 +650,7 @@ // If the map entry for the object was never marked as containing attached // pointers, no need to do any checking. - if (!TPR.Entry || !TPR.Entry->getMayContainAttachedPointers()) + if (!TPR.getEntry() || !TPR.getEntry()->getMayContainAttachedPointers()) return; uintptr_t LB = (uintptr_t)Begin; @@ -729,7 +732,6 @@ } } - bool IsLast, IsHostPtr; bool IsImplicit = ArgTypes[I] & OMP_TGT_MAPTYPE_IMPLICIT; bool UpdateRef = (!(ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) || (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)) && @@ -739,10 +741,10 @@ bool HasHoldModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_OMPX_HOLD; // If PTR_AND_OBJ, HstPtrBegin is address of pointee - TargetPointerResultTy TPR = Device.getTgtPtrBegin( - HstPtrBegin, DataSize, IsLast, UpdateRef, HasHoldModifier, IsHostPtr, - !IsImplicit, ForceDelete); - void *TgtPtrBegin = TPR.TargetPointer; + TargetPointerResultTy TPR = + Device.getTgtPtrBegin(HstPtrBegin, DataSize, UpdateRef, HasHoldModifier, + !IsImplicit, ForceDelete); + void *TgtPtrBegin = TPR.getTargetPointer(); if (!TgtPtrBegin && (DataSize || HasPresentModifier)) { DP("Mapping does not exist (%s)\n", (HasPresentModifier ? "'present' map type modifier" : "ignored")); @@ -770,7 +772,7 @@ } else { DP("There are %" PRId64 " bytes allocated at target address " DPxMOD " - is%s last\n", - DataSize, DPxPTR(TgtPtrBegin), (IsLast ? "" : " not")); + DataSize, DPxPTR(TgtPtrBegin), (TPR.isLast() ? "" : " not")); } // OpenMP 5.1, sec. 2.21.7.1 "map Clause", p. 351 L14-16: @@ -781,7 +783,7 @@ if (!TgtPtrBegin) continue; - bool DelEntry = IsLast; + bool DelEntry = TPR.isLast(); // If the last element from the mapper (for end transfer args comes in // reverse order), do not remove the partial entry, the parent struct still @@ -795,13 +797,12 @@ // Move data back to the host if (ArgTypes[I] & OMP_TGT_MAPTYPE_FROM) { bool Always = ArgTypes[I] & OMP_TGT_MAPTYPE_ALWAYS; - if ((Always || IsLast) && !IsHostPtr) { + if ((Always || TPR.isLast()) && !TPR.isHostPtr()) { DP("Moving %" PRId64 " bytes (tgt:" DPxMOD ") -> (hst:" DPxMOD ")\n", DataSize, DPxPTR(TgtPtrBegin), DPxPTR(HstPtrBegin)); - std::lock_guard LG(*TPR.Entry); // Wait for any previous transfer if an event is present. - if (void *Event = TPR.Entry->getEvent()) { + if (void *Event = TPR.getEntry()->getEvent()) { if (Device.waitEvent(Event, AsyncInfo) != OFFLOAD_SUCCESS) { REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event)); return OFFLOAD_FAIL; @@ -820,8 +821,8 @@ // as the entry can be reused and the reuse might happen after the // copy-back was issued but before it completed. Since the reuse might // also copy-back a value we would race. - if (IsLast) { - if (TPR.Entry->addEventIfNecessary(Device, AsyncInfo) != + if (TPR.isLast()) { + if (TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) != OFFLOAD_SUCCESS) return OFFLOAD_FAIL; } @@ -834,7 +835,7 @@ // Add pointer to the buffer for post-synchronize processing. PostProcessingPtrs.emplace_back(HstPtrBegin, DataSize, ArgTypes[I], - DelEntry && !IsHostPtr, TPR); + DelEntry && !TPR.isHostPtr(), TPR); } } @@ -913,11 +914,10 @@ void *HstPtrBegin, int64_t ArgSize, int64_t ArgType, AsyncInfoTy &AsyncInfo) { TIMESCOPE_WITH_IDENT(loc); - bool IsLast, IsHostPtr; - TargetPointerResultTy TPR = Device.getTgtPtrBegin( - HstPtrBegin, ArgSize, IsLast, /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false, IsHostPtr, /*MustContain=*/true); - void *TgtPtrBegin = TPR.TargetPointer; + TargetPointerResultTy TPR = + Device.getTgtPtrBegin(HstPtrBegin, ArgSize, /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false, /*MustContain=*/true); + void *TgtPtrBegin = TPR.getTargetPointer(); if (!TgtPtrBegin) { DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin)); if (ArgType & OMP_TGT_MAPTYPE_PRESENT) { @@ -929,7 +929,7 @@ return OFFLOAD_SUCCESS; } - if (IsHostPtr) { + if (TPR.isHostPtr()) { DP("hst data:" DPxMOD " unified and shared, becomes a noop\n", DPxPTR(HstPtrBegin)); return OFFLOAD_SUCCESS; @@ -1356,7 +1356,6 @@ void *HstPtrVal = Args[I]; void *HstPtrBegin = ArgBases[I]; void *HstPtrBase = Args[Idx]; - bool IsLast, IsHostPtr; // IsLast is unused. void *TgtPtrBase = (void *)((intptr_t)TgtArgs[TgtIdx] + TgtOffsets[TgtIdx]); DP("Parent lambda base " DPxMOD "\n", DPxPTR(TgtPtrBase)); @@ -1364,15 +1363,15 @@ void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta); void *&PointerTgtPtrBegin = AsyncInfo.getVoidPtrLocation(); TargetPointerResultTy TPR = Device.getTgtPtrBegin( - HstPtrVal, ArgSizes[I], IsLast, /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false, IsHostPtr); - PointerTgtPtrBegin = TPR.TargetPointer; + HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false, + /*UseHoldRefCount=*/false); + PointerTgtPtrBegin = TPR.getTargetPointer(); if (!PointerTgtPtrBegin) { DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n", DPxPTR(HstPtrVal)); continue; } - if (IsHostPtr) { + if (TPR.isHostPtr()) { DP("Unified memory is active, no need to map lambda captured" "variable (" DPxMOD ")\n", DPxPTR(HstPtrVal)); @@ -1394,7 +1393,6 @@ void *TgtPtrBegin; map_var_info_t HstPtrName = (!ArgNames) ? nullptr : ArgNames[I]; ptrdiff_t TgtBaseOffset; - bool IsLast, IsHostPtr; // unused. TargetPointerResultTy TPR; if (ArgTypes[I] & OMP_TGT_MAPTYPE_LITERAL) { DP("Forwarding first-private value " DPxMOD " to the target construct\n", @@ -1421,10 +1419,10 @@ } else { if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) HstPtrBase = *reinterpret_cast(HstPtrBase); - TPR = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], IsLast, + TPR = Device.getTgtPtrBegin(HstPtrBegin, ArgSizes[I], /*UpdateRefCount=*/false, - /*UseHoldRefCount=*/false, IsHostPtr); - TgtPtrBegin = TPR.TargetPointer; + /*UseHoldRefCount=*/false); + TgtPtrBegin = TPR.getTargetPointer(); TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin; #ifdef OMPTARGET_DEBUG void *TgtPtrBase = (void *)((intptr_t)TgtPtrBegin + TgtBaseOffset);