This is an archive of the discontinued LLVM Phabricator instance.

Differential D55130

[OPENMP][NVPTX]Make runtime compatible with the original runtime.
ClosedPublic

Authored by ABataev on Nov 30 2018, 8:31 AM.

Details

Reviewers
gtbercea
kkwli0
Commits
rG0f221f53d80b: [OPENMP][NVPTX]Make runtime compatible with the original runtime.
rOMP348003: [OPENMP][NVPTX]Make runtime compatible with the original runtime.
rL348003: [OPENMP][NVPTX]Make runtime compatible with the original runtime.
Summary

Reworked runtime to make it compatible with the requirements of the
original runtime library. Also, simplified some code to reduce number of
function calls.

Diff Detail

Event Timeline

ABataev created this revision.Nov 30 2018, 8:31 AM
Herald added subscribers: jfb, guansong. · View Herald TranscriptNov 30 2018, 8:31 AM
Harbormaster completed remote builds in B25550: Diff 176134.Nov 30 2018, 8:31 AM
gtbercea accepted this revision.Nov 30 2018, 8:36 AM

LGTM

This revision is now accepted and ready to land.Nov 30 2018, 8:36 AM
Closed by commit rL348003: [OPENMP][NVPTX]Make runtime compatible with the original runtime. (authored by ABataev). · Explain WhyNov 30 2018, 8:55 AM
This revision was automatically updated to reflect the committed changes.
Herald added a subscriber: llvm-commits. · View Herald TranscriptNov 30 2018, 8:55 AM

Revision Contents

PathSize
libomptarget/
deviceRTLs/
nvptx/
src/
215 lines
4 lines
3 lines
15 lines
9 lines

Diff 176134

libomptarget/deviceRTLs/nvptx/src/loop.cu

Show First 20 Lines • Show All 87 Lines • ▼ Show 20 Lines INLINE static void ForStaticNoChunk(int &last, T &lb, T &ub, ST &stride,
ub = lb + chunk - 1; // Clang uses i <= ub ub = lb + chunk - 1; // Clang uses i <= ub
last = lb <= inputUb && inputUb <= ub; last = lb <= inputUb && inputUb <= ub;
stride = loopSize; // make sure we only do 1 chunk per warp stride = loopSize; // make sure we only do 1 chunk per warp
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Support for Static Init // Support for Static Init
INLINE static void for_static_init(int32_t schedtype, int32_t *plastiter, INLINE static void for_static_init(int32_t gtid, int32_t schedtype,
T *plower, T *pupper, ST *pstride, int32_t *plastiter, T *plower, T *pupper,
ST chunk, bool IsSPMDExecutionMode, ST *pstride, ST chunk,
bool IsSPMDExecutionMode,
bool IsRuntimeUninitialized) { bool IsRuntimeUninitialized) {
// When IsRuntimeUninitialized is true, we assume that the caller is // When IsRuntimeUninitialized is true, we assume that the caller is
// in an L0 parallel region and that all worker threads participate. // in an L0 parallel region and that all worker threads participate.
int tid = GetLogicalThreadIdInBlock(); int tid = GetLogicalThreadIdInBlock();
// Assume we are in teams region or that we use a single block // Assume we are in teams region or that we use a single block
// per target region // per target region
ST numberOfActiveOMPThreads = GetNumberOfOmpThreads( ST numberOfActiveOMPThreads = GetNumberOfOmpThreads(
tid, IsSPMDExecutionMode, IsRuntimeUninitialized); tid, IsSPMDExecutionMode, IsRuntimeUninitialized);
// All warps that are in excess of the maximum requested, do // All warps that are in excess of the maximum requested, do
// not execute the loop // not execute the loop
PRINT(LD_LOOP, PRINT(LD_LOOP,
"OMP Thread %d: schedule type %d, chunk size = %lld, mytid " "OMP Thread %d: schedule type %d, chunk size = %lld, mytid "
"%d, num tids %d\n", "%d, num tids %d\n",
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized), gtid, schedtype, P64(chunk), gtid, numberOfActiveOMPThreads);
schedtype, P64(chunk), ASSERT0(LT_FUSSY, gtid < numberOfActiveOMPThreads,
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized),
GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized));
ASSERT0(
LT_FUSSY,
(GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized)) <
(GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized)),
"current thread is not needed here; error"); "current thread is not needed here; error");
// copy // copy
int lastiter = 0; int lastiter = 0;
T lb = *plower; T lb = *plower;
T ub = *pupper; T ub = *pupper;
ST stride = *pstride; ST stride = *pstride;
T entityId, numberOfEntities;
// init // init
switch (SCHEDULE_WITHOUT_MODIFIERS(schedtype)) { switch (SCHEDULE_WITHOUT_MODIFIERS(schedtype)) {
case kmp_sched_static_chunk: { case kmp_sched_static_chunk: {
if (chunk > 0) { if (chunk > 0) {
entityId = ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized); numberOfActiveOMPThreads);
numberOfEntities = GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized);
ForStaticChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
break; break;
} }
} // note: if chunk <=0, use nochunk } // note: if chunk <=0, use nochunk
case kmp_sched_static_balanced_chunk: { case kmp_sched_static_balanced_chunk: {
if (chunk > 0) { if (chunk > 0) {
entityId =
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized);
numberOfEntities = GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized);
// round up to make sure the chunk is enough to cover all iterations // round up to make sure the chunk is enough to cover all iterations
T tripCount = ub - lb + 1; // +1 because ub is inclusive T tripCount = ub - lb + 1; // +1 because ub is inclusive
T span = (tripCount + numberOfEntities - 1) / numberOfEntities; T span = (tripCount + numberOfActiveOMPThreads - 1) /
numberOfActiveOMPThreads;
// perform chunk adjustment // perform chunk adjustment
chunk = (span + chunk - 1) & ~(chunk - 1); chunk = (span + chunk - 1) & ~(chunk - 1);
ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb."); ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb.");
T oldUb = ub; T oldUb = ub;
ForStaticChunk(lastiter, lb, ub, stride, chunk, entityId, ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
numberOfEntities); numberOfActiveOMPThreads);
if (ub > oldUb) if (ub > oldUb)
ub = oldUb; ub = oldUb;
break; break;
} }
} // note: if chunk <=0, use nochunk } // note: if chunk <=0, use nochunk
case kmp_sched_static_nochunk: { case kmp_sched_static_nochunk: {
entityId = ForStaticNoChunk(lastiter, lb, ub, stride, chunk, gtid,
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized); numberOfActiveOMPThreads);
numberOfEntities = GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized);
ForStaticNoChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
break; break;
} }
case kmp_sched_distr_static_chunk: { case kmp_sched_distr_static_chunk: {
if (chunk > 0) { if (chunk > 0) {
entityId = GetOmpTeamId(); ForStaticChunk(lastiter, lb, ub, stride, chunk, GetOmpTeamId(),
numberOfEntities = GetNumberOfOmpTeams(); GetNumberOfOmpTeams());
ForStaticChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
break; break;
} // note: if chunk <=0, use nochunk } // note: if chunk <=0, use nochunk
} }
case kmp_sched_distr_static_nochunk: { case kmp_sched_distr_static_nochunk: {
entityId = GetOmpTeamId(); ForStaticNoChunk(lastiter, lb, ub, stride, chunk, GetOmpTeamId(),
numberOfEntities = GetNumberOfOmpTeams(); GetNumberOfOmpTeams());
ForStaticNoChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
break; break;
} }
case kmp_sched_distr_static_chunk_sched_static_chunkone: { case kmp_sched_distr_static_chunk_sched_static_chunkone: {
entityId = ForStaticChunk(lastiter, lb, ub, stride, chunk,
GetNumberOfOmpThreads(tid, IsSPMDExecutionMode, numberOfActiveOMPThreads * GetOmpTeamId() + gtid,
IsRuntimeUninitialized) * GetNumberOfOmpTeams() * numberOfActiveOMPThreads);
GetOmpTeamId() +
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized);
numberOfEntities = GetNumberOfOmpTeams() *
GetNumberOfOmpThreads(tid, IsSPMDExecutionMode,
IsRuntimeUninitialized);
ForStaticChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
break; break;
} }
default: { default: {
ASSERT(LT_FUSSY, FALSE, "unknown schedtype %d", schedtype); ASSERT(LT_FUSSY, FALSE, "unknown schedtype %d", schedtype);
PRINT(LD_LOOP, "unknown schedtype %d, revert back to static chunk\n", PRINT(LD_LOOP, "unknown schedtype %d, revert back to static chunk\n",
schedtype); schedtype);
entityId = ForStaticChunk(lastiter, lb, ub, stride, chunk, gtid,
GetOmpThreadId(tid, IsSPMDExecutionMode, IsRuntimeUninitialized); numberOfActiveOMPThreads);
numberOfEntities = GetNumberOfOmpThreads(tid, IsSPMDExecutionMode, break;
IsRuntimeUninitialized);
ForStaticChunk(lastiter, lb, ub, stride, chunk, entityId,
numberOfEntities);
} }
} }
// copy back // copy back
*plastiter = lastiter; *plastiter = lastiter;
*plower = lb; *plower = lb;
*pupper = ub; *pupper = ub;
*pstride = stride; *pstride = stride;
PRINT( PRINT(LD_LOOP,
LD_LOOP,
"Got sched: Active %d, total %d: lb %lld, ub %lld, stride %lld, last " "Got sched: Active %d, total %d: lb %lld, ub %lld, stride %lld, last "
"%d\n", "%d\n",
GetNumberOfOmpThreads(tid, IsSPMDExecutionMode, IsRuntimeUninitialized), numberOfActiveOMPThreads, GetNumberOfWorkersInTeam(), P64(*plower),
GetNumberOfWorkersInTeam(), P64(*plower), P64(*pupper), P64(*pstride), P64(*pupper), P64(*pstride), lastiter);
lastiter);
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Support for dispatch Init // Support for dispatch Init
INLINE static int OrderedSchedule(kmp_sched_t schedule) { INLINE static int OrderedSchedule(kmp_sched_t schedule) {
return schedule >= kmp_sched_ordered_first && return schedule >= kmp_sched_ordered_first &&
schedule <= kmp_sched_ordered_last; schedule <= kmp_sched_ordered_last;
} }
INLINE static void dispatch_init(kmp_Ident *loc, int32_t threadId, INLINE static void dispatch_init(kmp_Ident *loc, int32_t threadId,
kmp_sched_t schedule, T lb, T ub, ST st, kmp_sched_t schedule, T lb, T ub, ST st,
ST chunk) { ST chunk) {
ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
"Expected non-SPMD mode + initialized runtime."); "Expected non-SPMD mode + initialized runtime.");
int tid = GetLogicalThreadIdInBlock(); int tid = GetLogicalThreadIdInBlock();
omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(tid); omptarget_nvptx_TaskDescr *currTaskDescr = getMyTopTaskDescriptor(tid);
T tnum = currTaskDescr->ThreadsInTeam(); T tnum = currTaskDescr->ThreadsInTeam();
T tripCount = ub - lb + 1; // +1 because ub is inclusive T tripCount = ub - lb + 1; // +1 because ub is inclusive
ASSERT0( ASSERT0(LT_FUSSY, threadId < tnum,
LT_FUSSY,
GetOmpThreadId(tid, checkSPMDMode(loc), checkRuntimeUninitialized(loc)) <
GetNumberOfOmpThreads(tid, checkSPMDMode(loc),
checkRuntimeUninitialized(loc)),
"current thread is not needed here; error"); "current thread is not needed here; error");
/* Currently just ignore the monotonic and non-monotonic modifiers /* Currently just ignore the monotonic and non-monotonic modifiers
* (the compiler isn't producing them * yet anyway). * (the compiler isn't producing them * yet anyway).
* When it is we'll want to look at them somewhere here and use that * When it is we'll want to look at them somewhere here and use that
* information to add to our schedule choice. We shouldn't need to pass * information to add to our schedule choice. We shouldn't need to pass
* them on, they merely affect which schedule we can legally choose for * them on, they merely affect which schedule we can legally choose for
* various dynamic cases. (In paritcular, whether or not a stealing scheme * various dynamic cases. (In paritcular, whether or not a stealing scheme
* is legal). * is legal).
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines if (schedule == kmp_sched_static_chunk) {
ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value"); ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value");
// save sched state // save sched state
omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule; omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
// save ub // save ub
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub; omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
// compute static chunk // compute static chunk
ST stride; ST stride;
int lastiter = 0; int lastiter = 0;
ForStaticChunk( ForStaticChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
lastiter, lb, ub, stride, chunk,
GetOmpThreadId(tid, checkSPMDMode(loc),
checkRuntimeUninitialized(loc)), tnum);
// save computed params // save computed params
omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk; omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb; omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
omptarget_nvptx_threadPrivateContext->Stride(tid) = stride; omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
PRINT(LD_LOOP, PRINT(LD_LOOP,
"dispatch init (static chunk) : num threads = %d, ub = %" PRId64 "dispatch init (static chunk) : num threads = %d, ub = %" PRId64
", next lower bound = %llu, stride = %llu\n", ", next lower bound = %llu, stride = %llu\n",
GetNumberOfOmpThreads(tid, checkSPMDMode(loc), tnum, omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
checkRuntimeUninitialized(loc)),
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid), omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
omptarget_nvptx_threadPrivateContext->Stride(tid)); omptarget_nvptx_threadPrivateContext->Stride(tid));
} else if (schedule == kmp_sched_static_balanced_chunk) { } else if (schedule == kmp_sched_static_balanced_chunk) {
ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value"); ASSERT0(LT_FUSSY, chunk > 0, "bad chunk value");
// save sched state // save sched state
omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule; omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
// save ub // save ub
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub; omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
// compute static chunk // compute static chunk
ST stride; ST stride;
int lastiter = 0; int lastiter = 0;
// round up to make sure the chunk is enough to cover all iterations // round up to make sure the chunk is enough to cover all iterations
T span = (tripCount + tnum - 1) / tnum; T span = (tripCount + tnum - 1) / tnum;
// perform chunk adjustment // perform chunk adjustment
chunk = (span + chunk - 1) & ~(chunk - 1); chunk = (span + chunk - 1) & ~(chunk - 1);
T oldUb = ub; T oldUb = ub;
ForStaticChunk( ForStaticChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
lastiter, lb, ub, stride, chunk,
GetOmpThreadId(tid, checkSPMDMode(loc),
checkRuntimeUninitialized(loc)), tnum);
ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb."); ASSERT0(LT_FUSSY, ub >= lb, "ub must be >= lb.");
if (ub > oldUb) if (ub > oldUb)
ub = oldUb; ub = oldUb;
// save computed params // save computed params
omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk; omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb; omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
omptarget_nvptx_threadPrivateContext->Stride(tid) = stride; omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
PRINT(LD_LOOP, PRINT(LD_LOOP,
"dispatch init (static chunk) : num threads = %d, ub = %" PRId64 "dispatch init (static chunk) : num threads = %d, ub = %" PRId64
", next lower bound = %llu, stride = %llu\n", ", next lower bound = %llu, stride = %llu\n",
GetNumberOfOmpThreads(tid, checkSPMDMode(loc), tnum, omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
checkRuntimeUninitialized(loc)),
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid), omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
omptarget_nvptx_threadPrivateContext->Stride(tid)); omptarget_nvptx_threadPrivateContext->Stride(tid));
} else if (schedule == kmp_sched_static_nochunk) { } else if (schedule == kmp_sched_static_nochunk) {
ASSERT0(LT_FUSSY, chunk == 0, "bad chunk value"); ASSERT0(LT_FUSSY, chunk == 0, "bad chunk value");
// save sched state // save sched state
omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule; omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
// save ub // save ub
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub; omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid) = ub;
// compute static chunk // compute static chunk
ST stride; ST stride;
int lastiter = 0; int lastiter = 0;
ForStaticNoChunk( ForStaticNoChunk(lastiter, lb, ub, stride, chunk, threadId, tnum);
lastiter, lb, ub, stride, chunk,
GetOmpThreadId(tid, checkSPMDMode(loc),
checkRuntimeUninitialized(loc)), tnum);
// save computed params // save computed params
omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk; omptarget_nvptx_threadPrivateContext->Chunk(tid) = chunk;
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb; omptarget_nvptx_threadPrivateContext->NextLowerBound(tid) = lb;
omptarget_nvptx_threadPrivateContext->Stride(tid) = stride; omptarget_nvptx_threadPrivateContext->Stride(tid) = stride;
PRINT(LD_LOOP, PRINT(LD_LOOP,
"dispatch init (static nochunk) : num threads = %d, ub = %" PRId64 "dispatch init (static nochunk) : num threads = %d, ub = %" PRId64
", next lower bound = %llu, stride = %llu\n", ", next lower bound = %llu, stride = %llu\n",
GetNumberOfOmpThreads(tid, checkSPMDMode(loc), tnum, omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
checkRuntimeUninitialized(loc)),
omptarget_nvptx_threadPrivateContext->LoopUpperBound(tid),
omptarget_nvptx_threadPrivateContext->NextLowerBound(tid), omptarget_nvptx_threadPrivateContext->NextLowerBound(tid),
omptarget_nvptx_threadPrivateContext->Stride(tid)); omptarget_nvptx_threadPrivateContext->Stride(tid));
} else if (schedule == kmp_sched_dynamic || schedule == kmp_sched_guided) { } else if (schedule == kmp_sched_dynamic || schedule == kmp_sched_guided) {
__kmpc_barrier(loc, threadId); __kmpc_barrier(loc, threadId);
// save sched state // save sched state
int teamId = GetOmpTeamId(); int teamId = GetOmpTeamId();
omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule; omptarget_nvptx_threadPrivateContext->ScheduleType(tid) = schedule;
if (GetThreadIdInBlock() == 0) { if (GetThreadIdInBlock() == 0) {
if (chunk < 1) if (chunk < 1)
chunk = 1; chunk = 1;
omptarget_nvptx_threadPrivateContext->Chunk(teamId) = chunk; omptarget_nvptx_threadPrivateContext->Chunk(teamId) = chunk;
omptarget_nvptx_threadPrivateContext->LoopUpperBound(teamId) = ub; omptarget_nvptx_threadPrivateContext->LoopUpperBound(teamId) = ub;
omptarget_nvptx_threadPrivateContext->NextLowerBound(teamId) = lb; omptarget_nvptx_threadPrivateContext->NextLowerBound(teamId) = lb;
} }
__kmpc_barrier(loc, threadId); __kmpc_barrier(loc, threadId);
PRINT(LD_LOOP, PRINT(LD_LOOP,
"dispatch init (dyn) : num threads = %d, lb = %llu, ub = %" PRId64 "dispatch init (dyn) : num threads = %d, lb = %llu, ub = %" PRId64
", chunk %" PRIu64 "\n", ", chunk %" PRIu64 "\n",
GetNumberOfOmpThreads(tid, checkSPMDMode(loc), tnum, omptarget_nvptx_threadPrivateContext->NextLowerBound(teamId),
checkRuntimeUninitialized(loc)),
omptarget_nvptx_threadPrivateContext->NextLowerBound(teamId),
omptarget_nvptx_threadPrivateContext->LoopUpperBound(teamId), omptarget_nvptx_threadPrivateContext->LoopUpperBound(teamId),
omptarget_nvptx_threadPrivateContext->Chunk(teamId)); omptarget_nvptx_threadPrivateContext->Chunk(teamId));
} }
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Support for dispatch next // Support for dispatch next
Show All 29 Lines INLINE static int DynamicNextChunk(T &lb, T &ub, T chunkSize,
PRINT(LD_LOOPD, "lb %lld, ub %lld, loop ub %lld; finished\n", P64(lb), PRINT(LD_LOOPD, "lb %lld, ub %lld, loop ub %lld; finished\n", P64(lb),
P64(ub), P64(loopUpperBound)); P64(ub), P64(loopUpperBound));
return FINISHED; return FINISHED;
} }
// On Pascal, with inlining of the runtime into the user application, // On Pascal, with inlining of the runtime into the user application,
// this code deadlocks. This is probably because different threads // this code deadlocks. This is probably because different threads
// in a warp cannot make independent progress. // in a warp cannot make independent progress.
NOINLINE static int dispatch_next(int32_t *plast, T *plower, T *pupper, NOINLINE static int dispatch_next(int32_t gtid, int32_t *plast, T *plower,
ST *pstride) { T *pupper, ST *pstride) {
ASSERT0(LT_FUSSY, isRuntimeInitialized(), ASSERT0(LT_FUSSY, isRuntimeInitialized(),
"Expected non-SPMD mode + initialized runtime."); "Expected non-SPMD mode + initialized runtime.");
// ID of a thread in its own warp // ID of a thread in its own warp
// automatically selects thread or warp ID based on selected implementation // automatically selects thread or warp ID based on selected implementation
int tid = GetLogicalThreadIdInBlock(); int tid = GetLogicalThreadIdInBlock();
ASSERT0( ASSERT0(LT_FUSSY,
LT_FUSSY, gtid < GetNumberOfOmpThreads(tid, isSPMDMode(),
GetOmpThreadId(tid, isSPMDMode(), isRuntimeUninitialized()) < isRuntimeUninitialized()),
GetNumberOfOmpThreads(tid, isSPMDMode(), isRuntimeUninitialized()),
"current thread is not needed here; error"); "current thread is not needed here; error");
// retrieve schedule // retrieve schedule
kmp_sched_t schedule = kmp_sched_t schedule =
omptarget_nvptx_threadPrivateContext->ScheduleType(tid); omptarget_nvptx_threadPrivateContext->ScheduleType(tid);
// xxx reduce to one // xxx reduce to one
if (schedule == kmp_sched_static_chunk || if (schedule == kmp_sched_static_chunk ||
schedule == kmp_sched_static_nochunk) { schedule == kmp_sched_static_nochunk) {
T myLb = omptarget_nvptx_threadPrivateContext->NextLowerBound(tid); T myLb = omptarget_nvptx_threadPrivateContext->NextLowerBound(tid);
▲ Show 20 Lines • Show All 94 Lines • ▼ Show 20 Lines omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_init(
loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk); loc, tid, (kmp_sched_t)schedule, lb, ub, st, chunk);
} }
// next // next
EXTERN int __kmpc_dispatch_next_4(kmp_Ident *loc, int32_t tid, int32_t *p_last, EXTERN int __kmpc_dispatch_next_4(kmp_Ident *loc, int32_t tid, int32_t *p_last,
int32_t *p_lb, int32_t *p_ub, int32_t *p_st) { int32_t *p_lb, int32_t *p_ub, int32_t *p_st) {
PRINT0(LD_IO, "call kmpc_dispatch_next_4\n"); PRINT0(LD_IO, "call kmpc_dispatch_next_4\n");
return omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_next( return omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_next(
p_last, p_lb, p_ub, p_st); tid, p_last, p_lb, p_ub, p_st);
} }
EXTERN int __kmpc_dispatch_next_4u(kmp_Ident *loc, int32_t tid, EXTERN int __kmpc_dispatch_next_4u(kmp_Ident *loc, int32_t tid,
int32_t *p_last, uint32_t *p_lb, int32_t *p_last, uint32_t *p_lb,
uint32_t *p_ub, int32_t *p_st) { uint32_t *p_ub, int32_t *p_st) {
PRINT0(LD_IO, "call kmpc_dispatch_next_4u\n"); PRINT0(LD_IO, "call kmpc_dispatch_next_4u\n");
return omptarget_nvptx_LoopSupport<uint32_t, int32_t>::dispatch_next( return omptarget_nvptx_LoopSupport<uint32_t, int32_t>::dispatch_next(
p_last, p_lb, p_ub, p_st); tid, p_last, p_lb, p_ub, p_st);
} }
EXTERN int __kmpc_dispatch_next_8(kmp_Ident *loc, int32_t tid, int32_t *p_last, EXTERN int __kmpc_dispatch_next_8(kmp_Ident *loc, int32_t tid, int32_t *p_last,
int64_t *p_lb, int64_t *p_ub, int64_t *p_st) { int64_t *p_lb, int64_t *p_ub, int64_t *p_st) {
PRINT0(LD_IO, "call kmpc_dispatch_next_8\n"); PRINT0(LD_IO, "call kmpc_dispatch_next_8\n");
return omptarget_nvptx_LoopSupport<int64_t, int64_t>::dispatch_next( return omptarget_nvptx_LoopSupport<int64_t, int64_t>::dispatch_next(
p_last, p_lb, p_ub, p_st); tid, p_last, p_lb, p_ub, p_st);
} }
EXTERN int __kmpc_dispatch_next_8u(kmp_Ident *loc, int32_t tid, EXTERN int __kmpc_dispatch_next_8u(kmp_Ident *loc, int32_t tid,
int32_t *p_last, uint64_t *p_lb, int32_t *p_last, uint64_t *p_lb,
uint64_t *p_ub, int64_t *p_st) { uint64_t *p_ub, int64_t *p_st) {
PRINT0(LD_IO, "call kmpc_dispatch_next_8u\n"); PRINT0(LD_IO, "call kmpc_dispatch_next_8u\n");
return omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_next( return omptarget_nvptx_LoopSupport<uint64_t, int64_t>::dispatch_next(
p_last, p_lb, p_ub, p_st); tid, p_last, p_lb, p_ub, p_st);
} }
// fini // fini
EXTERN void __kmpc_dispatch_fini_4(kmp_Ident *loc, int32_t tid) { EXTERN void __kmpc_dispatch_fini_4(kmp_Ident *loc, int32_t tid) {
PRINT0(LD_IO, "call kmpc_dispatch_fini_4\n"); PRINT0(LD_IO, "call kmpc_dispatch_fini_4\n");
omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_fini(); omptarget_nvptx_LoopSupport<int32_t, int32_t>::dispatch_fini();
} }
Show All 18 Lines
EXTERN void __kmpc_for_static_init_4(kmp_Ident *loc, int32_t global_tid, EXTERN void __kmpc_for_static_init_4(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
int32_t *plower, int32_t *pupper, int32_t *plower, int32_t *pupper,
int32_t *pstride, int32_t incr, int32_t *pstride, int32_t incr,
int32_t chunk) { int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4\n");
omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
checkSPMDMode(loc), checkRuntimeUninitialized(loc)); checkSPMDMode(loc), checkRuntimeUninitialized(loc));
} }
EXTERN void __kmpc_for_static_init_4u(kmp_Ident *loc, int32_t global_tid, EXTERN void __kmpc_for_static_init_4u(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
uint32_t *plower, uint32_t *pupper, uint32_t *plower, uint32_t *pupper,
int32_t *pstride, int32_t incr, int32_t *pstride, int32_t incr,
int32_t chunk) { int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4u\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4u\n");
omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
checkSPMDMode(loc), checkRuntimeUninitialized(loc)); checkSPMDMode(loc), checkRuntimeUninitialized(loc));
} }
EXTERN void __kmpc_for_static_init_8(kmp_Ident *loc, int32_t global_tid, EXTERN void __kmpc_for_static_init_8(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
int64_t *plower, int64_t *pupper, int64_t *plower, int64_t *pupper,
int64_t *pstride, int64_t incr, int64_t *pstride, int64_t incr,
int64_t chunk) { int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8\n");
omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
checkSPMDMode(loc), checkRuntimeUninitialized(loc)); checkSPMDMode(loc), checkRuntimeUninitialized(loc));
} }
EXTERN void __kmpc_for_static_init_8u(kmp_Ident *loc, int32_t global_tid, EXTERN void __kmpc_for_static_init_8u(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
uint64_t *plower, uint64_t *pupper, uint64_t *plower, uint64_t *pupper,
int64_t *pstride, int64_t incr, int64_t *pstride, int64_t incr,
int64_t chunk) { int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8u\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8u\n");
omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
checkSPMDMode(loc), checkRuntimeUninitialized(loc)); checkSPMDMode(loc), checkRuntimeUninitialized(loc));
} }
EXTERN EXTERN
void __kmpc_for_static_init_4_simple_spmd(kmp_Ident *loc, int32_t global_tid, void __kmpc_for_static_init_4_simple_spmd(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
int32_t *plower, int32_t *pupper, int32_t *plower, int32_t *pupper,
int32_t *pstride, int32_t incr, int32_t *pstride, int32_t incr,
int32_t chunk) { int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_spmd\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_spmd\n");
omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/true, /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_4u_simple_spmd(kmp_Ident *loc, int32_t global_tid, void __kmpc_for_static_init_4u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t schedtype,
int32_t *plastiter, uint32_t *plower, int32_t *plastiter, uint32_t *plower,
uint32_t *pupper, int32_t *pstride, uint32_t *pupper, int32_t *pstride,
int32_t incr, int32_t chunk) { int32_t incr, int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_spmd\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_spmd\n");
omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/true, /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_8_simple_spmd(kmp_Ident *loc, int32_t global_tid, void __kmpc_for_static_init_8_simple_spmd(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t *plastiter, int32_t schedtype, int32_t *plastiter,
int64_t *plower, int64_t *pupper, int64_t *plower, int64_t *pupper,
int64_t *pstride, int64_t incr, int64_t *pstride, int64_t incr,
int64_t chunk) { int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_spmd\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_spmd\n");
omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/true, /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_8u_simple_spmd(kmp_Ident *loc, int32_t global_tid, void __kmpc_for_static_init_8u_simple_spmd(kmp_Ident *loc, int32_t global_tid,
int32_t schedtype, int32_t schedtype,
int32_t *plastiter, uint64_t *plower, int32_t *plastiter, uint64_t *plower,
uint64_t *pupper, int64_t *pstride, uint64_t *pupper, int64_t *pstride,
int64_t incr, int64_t chunk) { int64_t incr, int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_spmd\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_spmd\n");
omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/true, /*IsSPMDExecutionMode=*/true, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_4_simple_generic( void __kmpc_for_static_init_4_simple_generic(
kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter, kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
int32_t *plower, int32_t *pupper, int32_t *pstride, int32_t incr, int32_t *plower, int32_t *pupper, int32_t *pstride, int32_t incr,
int32_t chunk) { int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_generic\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4_simple_generic\n");
omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<int32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/false, /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_4u_simple_generic( void __kmpc_for_static_init_4u_simple_generic(
kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter, kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
uint32_t *plower, uint32_t *pupper, int32_t *pstride, int32_t incr, uint32_t *plower, uint32_t *pupper, int32_t *pstride, int32_t incr,
int32_t chunk) { int32_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_generic\n"); PRINT0(LD_IO, "call kmpc_for_static_init_4u_simple_generic\n");
omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init( omptarget_nvptx_LoopSupport<uint32_t, int32_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/false, /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_8_simple_generic( void __kmpc_for_static_init_8_simple_generic(
kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter, kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
int64_t *plower, int64_t *pupper, int64_t *pstride, int64_t incr, int64_t *plower, int64_t *pupper, int64_t *pstride, int64_t incr,
int64_t chunk) { int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_generic\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8_simple_generic\n");
omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<int64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/false, /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN EXTERN
void __kmpc_for_static_init_8u_simple_generic( void __kmpc_for_static_init_8u_simple_generic(
kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter, kmp_Ident *loc, int32_t global_tid, int32_t schedtype, int32_t *plastiter,
uint64_t *plower, uint64_t *pupper, int64_t *pstride, int64_t incr, uint64_t *plower, uint64_t *pupper, int64_t *pstride, int64_t incr,
int64_t chunk) { int64_t chunk) {
PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_generic\n"); PRINT0(LD_IO, "call kmpc_for_static_init_8u_simple_generic\n");
omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init( omptarget_nvptx_LoopSupport<uint64_t, int64_t>::for_static_init(
schedtype, plastiter, plower, pupper, pstride, chunk, global_tid, schedtype, plastiter, plower, pupper, pstride, chunk,
/*IsSPMDExecutionMode=*/false, /*IsSPMDExecutionMode=*/false, /*IsRuntimeUninitialized=*/true);
/*IsRuntimeUninitialized=*/true);
} }
EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid) { EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid) {
PRINT0(LD_IO, "call kmpc_for_static_fini\n"); PRINT0(LD_IO, "call kmpc_for_static_fini\n");
} }
namespace { namespace {
INLINE void syncWorkersInGenericMode(uint32_t NumThreads) { INLINE void syncWorkersInGenericMode(uint32_t NumThreads) {
Show All 15 Lines
EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc, int32_t gtid, EXTERN void __kmpc_reduce_conditional_lastprivate(kmp_Ident *loc, int32_t gtid,
int32_t varNum, void *array) { int32_t varNum, void *array) {
PRINT0(LD_IO, "call to __kmpc_reduce_conditional_lastprivate(...)\n"); PRINT0(LD_IO, "call to __kmpc_reduce_conditional_lastprivate(...)\n");
ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
"Expected non-SPMD mode + initialized runtime."); "Expected non-SPMD mode + initialized runtime.");
omptarget_nvptx_TeamDescr &teamDescr = getMyTeamDescriptor(); omptarget_nvptx_TeamDescr &teamDescr = getMyTeamDescriptor();
int tid = GetOmpThreadId(GetLogicalThreadIdInBlock(), checkSPMDMode(loc), int tid = GetLogicalThreadIdInBlock();
checkRuntimeUninitialized(loc)); uint32_t NumThreads = GetNumberOfOmpThreads(tid, checkSPMDMode(loc),
uint32_t NumThreads = GetNumberOfOmpThreads(
GetLogicalThreadIdInBlock(), checkSPMDMode(loc),
checkRuntimeUninitialized(loc)); checkRuntimeUninitialized(loc));
uint64_t *Buffer = teamDescr.getLastprivateIterBuffer(); uint64_t *Buffer = teamDescr.getLastprivateIterBuffer();
for (unsigned i = 0; i < varNum; i++) { for (unsigned i = 0; i < varNum; i++) {
// Reset buffer. // Reset buffer.
if (tid == 0) if (gtid == 0)
*Buffer = 0; // Reset to minimum loop iteration value. *Buffer = 0; // Reset to minimum loop iteration value.
// Barrier. // Barrier.
syncWorkersInGenericMode(NumThreads); syncWorkersInGenericMode(NumThreads);
// Atomic max of iterations. // Atomic max of iterations.
uint64_t *varArray = (uint64_t *)array; uint64_t *varArray = (uint64_t *)array;
uint64_t elem = varArray[i]; uint64_t elem = varArray[i];
Show All 13 Lines

libomptarget/deviceRTLs/nvptx/src/parallel.cu

Show First 20 Lines • Show All 412 Lines • ▼ Show 20 Lines else
return 0; return 0;
} }
// This kmpc call returns the thread id across all teams. It's value is // This kmpc call returns the thread id across all teams. It's value is
// cached by the compiler and used when calling the runtime. On nvptx // cached by the compiler and used when calling the runtime. On nvptx
// it's cheap to recalculate this value so we never use the result // it's cheap to recalculate this value so we never use the result
// of this call. // of this call.
EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc) { EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc) {
return GetLogicalThreadIdInBlock(); int tid = GetLogicalThreadIdInBlock();
return GetOmpThreadId(tid, checkSPMDMode(loc),
checkRuntimeUninitialized(loc));
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// push params // push params
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t tid, EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t tid,
int32_t num_threads) { int32_t num_threads) {
Show All 29 Lines

libomptarget/deviceRTLs/nvptx/src/reduction.cu

Show First 20 Lines • Show All 226 Lines • ▼ Show 20 Lines if (NumThreads > WARPSIZE) {
return BlockThreadId == 0; return BlockThreadId == 0;
} else if (isRuntimeUninitialized /* Never an L2 parallel region without the OMP runtime */) { } else if (isRuntimeUninitialized /* Never an L2 parallel region without the OMP runtime */) {
return BlockThreadId == 0; return BlockThreadId == 0;
} }
// Get the OMP thread Id. This is different from BlockThreadId in the case of // Get the OMP thread Id. This is different from BlockThreadId in the case of
// an L2 parallel region. // an L2 parallel region.
return GetOmpThreadId(BlockThreadId, isSPMDExecutionMode, return global_tid == 0;
isRuntimeUninitialized) == 0;
#endif // __CUDA_ARCH__ >= 700 #endif // __CUDA_ARCH__ >= 700
} }
EXTERN EXTERN
int32_t __kmpc_nvptx_parallel_reduce_nowait( int32_t __kmpc_nvptx_parallel_reduce_nowait(
int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data, int32_t global_tid, int32_t num_vars, size_t reduce_size, void *reduce_data,
kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) { kmp_ShuffleReductFctPtr shflFct, kmp_InterWarpCopyFctPtr cpyFct) {
return nvptx_parallel_reduce_nowait( return nvptx_parallel_reduce_nowait(
▲ Show 20 Lines • Show All 206 Lines • Show Last 20 Lines

libomptarget/deviceRTLs/nvptx/src/sync.cu

Show First 20 Lines • Show All 93 Lines • ▼ Show 20 LinesEXTERN void __kmpc_barrier_simple_generic(kmp_Ident *loc_ref, int32_t tid) {
named_sync(L1_BARRIER, threads); named_sync(L1_BARRIER, threads);
PRINT0(LD_SYNC, "completed kmpc_barrier_simple_generic\n"); PRINT0(LD_SYNC, "completed kmpc_barrier_simple_generic\n");
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// KMP MASTER // KMP MASTER
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
INLINE int32_t IsMaster() {
// only the team master updates the state
int tid = GetLogicalThreadIdInBlock();
int ompThreadId = GetOmpThreadId(tid, isSPMDMode(), isRuntimeUninitialized());
return IsTeamMaster(ompThreadId);
}
EXTERN int32_t __kmpc_master(kmp_Ident *loc, int32_t global_tid) { EXTERN int32_t __kmpc_master(kmp_Ident *loc, int32_t global_tid) {
PRINT0(LD_IO, "call kmpc_master\n"); PRINT0(LD_IO, "call kmpc_master\n");
return IsMaster(); return IsTeamMaster(global_tid);
} }
EXTERN void __kmpc_end_master(kmp_Ident *loc, int32_t global_tid) { EXTERN void __kmpc_end_master(kmp_Ident *loc, int32_t global_tid) {
PRINT0(LD_IO, "call kmpc_end_master\n"); PRINT0(LD_IO, "call kmpc_end_master\n");
ASSERT0(LT_FUSSY, IsMaster(), "expected only master here"); ASSERT0(LT_FUSSY, IsTeamMaster(global_tid), "expected only master here");
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// KMP SINGLE // KMP SINGLE
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
EXTERN int32_t __kmpc_single(kmp_Ident *loc, int32_t global_tid) { EXTERN int32_t __kmpc_single(kmp_Ident *loc, int32_t global_tid) {
PRINT0(LD_IO, "call kmpc_single\n"); PRINT0(LD_IO, "call kmpc_single\n");
// decide to implement single with master; master get the single // decide to implement single with master; master get the single
return IsMaster(); return IsTeamMaster(global_tid);
} }
EXTERN void __kmpc_end_single(kmp_Ident *loc, int32_t global_tid) { EXTERN void __kmpc_end_single(kmp_Ident *loc, int32_t global_tid) {
PRINT0(LD_IO, "call kmpc_end_single\n"); PRINT0(LD_IO, "call kmpc_end_single\n");
// decide to implement single with master: master get the single // decide to implement single with master: master get the single
ASSERT0(LT_FUSSY, IsMaster(), "expected only master here"); ASSERT0(LT_FUSSY, IsTeamMaster(global_tid), "expected only master here");
// sync barrier is explicitely called... so that is not a problem // sync barrier is explicitely called... so that is not a problem
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
// Flush // Flush
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
EXTERN void __kmpc_flush(kmp_Ident *loc) { EXTERN void __kmpc_flush(kmp_Ident *loc) {
Show All 12 Lines

libomptarget/deviceRTLs/nvptx/src/task.cu

Show First 20 Lines • Show All 75 Lines • ▼ Show 20 Lines
EXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid, EXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid,
kmp_TaskDescr *newKmpTaskDescr, kmp_TaskDescr *newKmpTaskDescr,
int32_t depNum, void *depList, int32_t depNum, void *depList,
int32_t noAliasDepNum, int32_t noAliasDepNum,
void *noAliasDepList) { void *noAliasDepList) {
PRINT(LD_IO, "call to __kmpc_omp_task_with_deps(task 0x%llx)\n", PRINT(LD_IO, "call to __kmpc_omp_task_with_deps(task 0x%llx)\n",
P64(newKmpTaskDescr)); P64(newKmpTaskDescr));
ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized."); ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
"Runtime must be initialized.");
// 1. get explict task descr from kmp task descr // 1. get explict task descr from kmp task descr
omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr = omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
(omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES( (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr)); newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr, ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
"bad assumptions"); "bad assumptions");
omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr; omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr, ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
Show All 20 LinesEXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid,
SafeFree(newExplicitTaskDescr, "explicit task descriptor"); SafeFree(newExplicitTaskDescr, "explicit task descriptor");
return 0; return 0;
} }
EXTERN void __kmpc_omp_task_begin_if0(kmp_Ident *loc, uint32_t global_tid, EXTERN void __kmpc_omp_task_begin_if0(kmp_Ident *loc, uint32_t global_tid,
kmp_TaskDescr *newKmpTaskDescr) { kmp_TaskDescr *newKmpTaskDescr) {
PRINT(LD_IO, "call to __kmpc_omp_task_begin_if0(task 0x%llx)\n", PRINT(LD_IO, "call to __kmpc_omp_task_begin_if0(task 0x%llx)\n",
P64(newKmpTaskDescr)); P64(newKmpTaskDescr));
ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized."); ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
"Runtime must be initialized.");
// 1. get explict task descr from kmp task descr // 1. get explict task descr from kmp task descr
omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr = omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
(omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES( (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr)); newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr, ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
"bad assumptions"); "bad assumptions");
omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr; omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr, ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
"bad assumptions"); "bad assumptions");
// 2. push new context: update new task descriptor // 2. push new context: update new task descriptor
int tid = GetLogicalThreadIdInBlock(); int tid = GetLogicalThreadIdInBlock();
omptarget_nvptx_TaskDescr *parentTaskDescr = getMyTopTaskDescriptor(tid); omptarget_nvptx_TaskDescr *parentTaskDescr = getMyTopTaskDescriptor(tid);
newTaskDescr->CopyForExplicitTask(parentTaskDescr); newTaskDescr->CopyForExplicitTask(parentTaskDescr);
// set new task descriptor as top // set new task descriptor as top
omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid, newTaskDescr); omptarget_nvptx_threadPrivateContext->SetTopLevelTaskDescr(tid, newTaskDescr);
// 3... noting to call... is inline // 3... noting to call... is inline
// 4 & 5 ... done in complete // 4 & 5 ... done in complete
} }
EXTERN void __kmpc_omp_task_complete_if0(kmp_Ident *loc, uint32_t global_tid, EXTERN void __kmpc_omp_task_complete_if0(kmp_Ident *loc, uint32_t global_tid,
kmp_TaskDescr *newKmpTaskDescr) { kmp_TaskDescr *newKmpTaskDescr) {
PRINT(LD_IO, "call to __kmpc_omp_task_complete_if0(task 0x%llx)\n", PRINT(LD_IO, "call to __kmpc_omp_task_complete_if0(task 0x%llx)\n",
P64(newKmpTaskDescr)); P64(newKmpTaskDescr));
ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc), "Runtime must be initialized."); ASSERT0(LT_FUSSY, checkRuntimeInitialized(loc),
"Runtime must be initialized.");
// 1. get explict task descr from kmp task descr // 1. get explict task descr from kmp task descr
omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr = omptarget_nvptx_ExplicitTaskDescr *newExplicitTaskDescr =
(omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES( (omptarget_nvptx_ExplicitTaskDescr *)SUB_BYTES(
newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr)); newKmpTaskDescr, sizeof(omptarget_nvptx_TaskDescr));
ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr, ASSERT0(LT_FUSSY, &newExplicitTaskDescr->kmpTaskDescr == newKmpTaskDescr,
"bad assumptions"); "bad assumptions");
omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr; omptarget_nvptx_TaskDescr *newTaskDescr = &newExplicitTaskDescr->taskDescr;
ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr, ASSERT0(LT_FUSSY, (uint64_t)newTaskDescr == (uint64_t)newExplicitTaskDescr,
▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines