Added support for OpenMP 4.5 simd modifier of the schedule clause of the OpenMP loop construct

[AndreyChurbanov]

Summary

In addition to schedule(simd:static) inplemented earlier (kmp_sch_static_balanced_chunked = 45) added new support for schedule(simd:guided) and schedule(simd:runtime):

kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */
kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */

These three schedules cannot be covered solely by compiler or runtime. New schedule types introduced so that the runtime library can get needed information from the compiler on the simd length (for runtime schedule) or on the need of chunk size adjustment (for guided schedule).

For simd:runtime the compiler should provide the simd length in chunk parameter (unused for non-simd schedule).

For simd:guided the runtime library should adjust the size of each chunk to be multiple of the given chunk size provided by compiler (which in turn should be multiple of simd length).

[jlpeyton]

runtime/test/worksharing/for/kmp_sch_simd.c

361–362

Can we add an array of schedules to iterate over:
{kmp_sch_guided_simd, kmp_sch_static_balanced_chunked} and then send those in to run_loop_64 and run_loop_32?

[AndreyChurbanov]

Fixed initial patch to work on more cases of setting simd schedules - specifying schedule(simd:runtime), then use environment variable or API to set the schedule.

Added more tests to check various ways of setting the schedule.

Fixed minor bug in kmp_runtime.cpp when chunk=0 could be propagated into dispatcher code.

[jlpeyton]

LGTM

runtime/src/kmp.h

enum sched_type {
     /* accessible only through KMP_SCHEDULE environment variable */
     kmp_sch_guided_iterative_chunked  = 42,
     kmp_sch_guided_analytical_chunked = 43,
 
     kmp_sch_static_steal              = 44,   /**< accessible only through KMP_SCHEDULE environment variable */
 
 #if OMP_45_ENABLED
     kmp_sch_static_balanced_chunked   = 45,   /**< static with chunk adjustment (e.g., simd) */
+    kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */
+    kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */
 #endif
 
     /* accessible only through KMP_SCHEDULE environment variable */
-    kmp_sch_upper                     = 46,   /**< upper bound for unordered values */
+    kmp_sch_upper                     = 48,   /**< upper bound for unordered values */
 
     kmp_ord_lower                     = 64,   /**< lower bound for ordered values, must be power of 2 */
     kmp_ord_static_chunked            = 65,
     kmp_ord_static                    = 66,   /**< ordered static unspecialized */
     kmp_ord_dynamic_chunked           = 67,
     kmp_ord_guided_chunked            = 68,
     kmp_ord_runtime                   = 69,
     kmp_ord_auto                      = 70,   /**< ordered auto */

 typedef struct KMP_ALIGN_CACHE dispatch_private_info {
     union private_info {
         dispatch_private_info32_t  p32;
         dispatch_private_info64_t  p64;
     } u;
     enum sched_type schedule;  /* scheduling algorithm */
     kmp_int32       ordered;   /* ordered clause specified */
     kmp_int32       ordered_bumped;
-    kmp_int32   ordered_dummy[KMP_MAX_ORDERED-3]; // to retain the structure size after making ordered_iteration scalar
+    // to retain the structure size after making ordered_iteration scalar
+    kmp_int32   ordered_dummy[KMP_MAX_ORDERED-3];
     struct dispatch_private_info * next; /* stack of buffers for nest of serial regions */
     kmp_int32       nomerge;   /* don't merge iters if serialized */
     kmp_int32       type_size; /* the size of types in private_info */
     enum cons_type  pushed_ws;
 } dispatch_private_info_t;
 
 typedef struct dispatch_shared_info32 {
     /* chunk index under dynamic, number of idle threads under static-steal;

runtime/src/kmp_dispatch.cpp

struct KMP_ALIGN_CACHE dispatch_private_info_template {
     // duplicate alignment here, otherwise size of structure is not correct in our compiler
     union KMP_ALIGN_CACHE private_info_tmpl {
         dispatch_private_infoXX_template< T > p;
         dispatch_private_info64_t             p64;
     } u;
     enum sched_type schedule;  /* scheduling algorithm */
     kmp_uint32      ordered;   /* ordered clause specified */
     kmp_uint32      ordered_bumped;
-    kmp_int32   ordered_dummy[KMP_MAX_ORDERED-3]; // to retain the structure size after making order
+    // to retain the structure size after making ordered_iteration scalar
+    kmp_int32   ordered_dummy[KMP_MAX_ORDERED-3];
     dispatch_private_info * next; /* stack of buffers for nest of serial regions */
     kmp_uint32      nomerge;   /* don't merge iters if serialized */
     kmp_uint32      type_size;
     enum cons_type  pushed_ws;
 };
 
 
 // replaces dispatch_shared_info{32,64} structures and dispatch_shared_info{32,64}_t types
#endif
             #endif
         }
 
         /* guided analytical not safe for too many threads */
         if ( schedule == kmp_sch_guided_analytical_chunked && th->th.th_team_nproc > 1<<20 ) {
             schedule = kmp_sch_guided_iterative_chunked;
             KMP_WARNING( DispatchManyThreads );
         }
+        if (schedule == kmp_sch_runtime_simd) {
+            // compiler provides simd_width in the chunk parameter
+            schedule = team->t.t_sched.r_sched_type;
+            // Detail the schedule if needed (global controls are differentiated appropriately)
+            if (schedule == kmp_sch_static) {
+                schedule = kmp_sch_static_balanced_chunked;
+            } else {
+                if ( schedule == kmp_sch_guided_chunked ) {
+                    schedule = kmp_sch_guided_simd;
+                }
+                chunk = team->t.t_sched.chunk * chunk;
+            }
+#if USE_ITT_BUILD
+            cur_chunk = chunk;
+#endif
+            #ifdef KMP_DEBUG
+            {
+                const char * buff;
+                // create format specifiers before the debug output
+                buff = __kmp_str_format(
+                    "__kmp_dispatch_init: T#%%d new: schedule:%%d chunk:%%%s\n",
+                    traits_t< ST >::spec );
+                KD_TRACE(10, ( buff, gtid, schedule, chunk ) );
+                __kmp_str_free( &buff );
+            }
+            #endif
+        }
         pr->u.p.parm1 = chunk;
     }
     KMP_ASSERT2( (kmp_sch_lower < schedule && schedule < kmp_sch_upper),
                 "unknown scheduling type" );
 
     pr->u.p.count = 0;
 
     if ( __kmp_env_consistency_check ) {
#endif
                 }
             }
             if ( pr->ordered ) {
                 pr->u.p.ordered_lower = init;
                 pr->u.p.ordered_upper = limit;
             }
             break;
         } // case
+    case kmp_sch_static_balanced_chunked:
+        { // similar to balanced, but chunk adjusted to multiple of simd width
+            T nth = th->th.th_team_nproc;
+            KD_TRACE(100,("__kmp_dispatch_init: T#%d runtime(simd:static)"
+                          " -> falling-through to static_greedy\n",gtid));
+            schedule = kmp_sch_static_greedy;
+            if ( nth > 1 )
+                pr->u.p.parm1 = ((tc+nth-1)/nth+chunk-1) & ~(chunk-1);
+            else
+                pr->u.p.parm1 = tc;
+            break;
+        } // case
     case kmp_sch_guided_iterative_chunked :
+    case kmp_sch_guided_simd:
         {
             T nproc = th->th.th_team_nproc;
             KD_TRACE(100,("__kmp_dispatch_init: T#%d kmp_sch_guided_iterative_chunked case\n",gtid));
 
             if ( nproc > 1 ) {
                 if ( (2L * chunk + 1 ) * nproc >= tc ) {
                     /* chunk size too large, switch to dynamic */
                     schedule = kmp_sch_dynamic_chunked;
#if USE_ITT_BUILD
             case kmp_sch_static_greedy:
                 cur_chunk = pr->u.p.parm1;
                 break;
             case kmp_sch_dynamic_chunked:
                 schedtype = 1;
                 break;
             case kmp_sch_guided_iterative_chunked:
             case kmp_sch_guided_analytical_chunked:
+            case kmp_sch_guided_simd:
                 schedtype = 2;
                 break;
             default:
 //            Should we put this case under "static"?
 //            case kmp_sch_static_steal:
                 schedtype = 3;
                 break;
             }
#endif
                         *p_lb = 0;
                         *p_ub = 0;
                         if ( p_st != NULL )
                             *p_st = 0;
                     } // if
                 } // case
                 break;
 
+            case kmp_sch_guided_simd:
+                { // same as iterative but curr-chunk adjusted to be multiple of given chunk
+                    T  chunk = pr->u.p.parm1;
+                    KD_TRACE(100,
+                        ("__kmp_dispatch_next: T#%d kmp_sch_guided_simd case\n",gtid));
+                    trip  = pr->u.p.tc;
+                    // Start atomic part of calculations
+                    while(1) {
+                        ST  remaining;             // signed, because can be < 0
+                        init = sh->u.s.iteration;  // shared value
+                        remaining = trip - init;
+                        if (remaining <= 0) { // AC: need to compare with 0 first
+                            status = 0; // nothing to do, don't try atomic op
+                            break;
+                        }
+                        KMP_DEBUG_ASSERT(init%chunk == 0);
+                        // compare with K*nproc*(chunk+1), K=2 by default
+                        if ((T)remaining < pr->u.p.parm2) {
+                            // use dynamic-style shcedule
+                            // atomically inrement iterations, get old value
+                            init = test_then_add<ST>((ST*)&sh->u.s.iteration, (ST)chunk);
+                            remaining = trip - init;
+                            if (remaining <= 0) {
+                                status = 0;    // all iterations got by other threads
+                            } else {
+                                // got some iterations to work on
+                                status = 1;
+                                if ( (T)remaining > chunk ) {
+                                    limit = init + chunk - 1;
+                                } else {
+                                    last = 1;   // the last chunk
+                                    limit = init + remaining - 1;
+                                } // if
+                            } // if
+                            break;
+                        } // if
+                        UT span = remaining * (*(double*)&pr->u.p.parm3); // divide by K*nproc
+                        UT rem = span % chunk;
+                        if (rem) // adjust so that span%chunk == 0
+                            span += chunk - rem;
+                        limit = init + span;
+                        if ( compare_and_swap<ST>( (ST*)&sh->u.s.iteration, (ST)init, (ST)limit ) ) {
+                            // CAS was successful, chunk obtained
+                            status = 1;
+                            --limit;
+                            break;
+                        } // if
+                    } // while
+                    if ( status != 0 ) {
+                        start = pr->u.p.lb;
+                        incr = pr->u.p.st;
+                        if ( p_st != NULL )
+                            *p_st = incr;
+                        *p_lb = start + init * incr;
+                        *p_ub = start + limit * incr;
+                        if ( pr->ordered ) {
+                            pr->u.p.ordered_lower = init;
+                            pr->u.p.ordered_upper = limit;
+                            #ifdef KMP_DEBUG
+                            {
+                                const char * buff;
+                                // create format specifiers before the debug output
+                                buff = __kmp_str_format(
+                                    "__kmp_dispatch_next: T#%%d ordered_lower:%%%s ordered_upper:%%%s\n",
+                                    traits_t< UT >::spec, traits_t< UT >::spec );
+                                KD_TRACE(1000, ( buff, gtid, pr->u.p.ordered_lower, pr->u.p.ordered_upper ) );
+                                __kmp_str_free( &buff );
+                            }
+                            #endif
+                        } // if
+                    } else {
+                        *p_lb = 0;
+                        *p_ub = 0;
+                        if ( p_st != NULL )
+                            *p_st = 0;
+                    } // if
+                } // case
+                break;
+
             case kmp_sch_guided_analytical_chunked:
                 {
                     T   chunkspec = pr->u.p.parm1;
                     UT chunkIdx;
     #if KMP_OS_WINDOWS && KMP_ARCH_X86
                     /* for storing original FPCW value for Windows* OS on
            IA-32 architecture 8-byte version */
                     unsigned int oldFpcw;

runtime/test/worksharing/for/kmp_sch_simd.c

+// RUN: %libomp-compile-and-run
+/*
+  Test for the 'schedule(simd:guided)' clause.
+  Compiler needs to generate a dynamic dispatching and pass the schedule
+  value 46 to the OpenMP RTL. Test uses numerous loop parameter combinations.
+*/
+#include <stdio.h>
+#include <omp.h>
+
+#if defined(WIN32) || defined(_WIN32)
+#include <windows.h>
+#define delay() Sleep(1);
+#else
+#include <unistd.h>
+#define delay() usleep(10);
+#endif
+
+// uncomment for debug diagnostics:
+//#define DEBUG
+
+#define SIMD_LEN 4
+
+// ---------------------------------------------------------------------------
+// Various definitions copied from OpenMP RTL
+enum sched {
+  kmp_sch_static_balanced_chunked = 45,
+  kmp_sch_guided_simd = 46,
+  kmp_sch_runtime_simd = 47,
+};
+typedef unsigned u32;
+typedef long long i64;
+typedef unsigned long long u64;
+typedef struct {
+  int reserved_1;
+  int flags;
+  int reserved_2;
+  int reserved_3;
+  char *psource;
+} id;
+
+extern int __kmpc_global_thread_num(id*);
+extern void __kmpc_barrier(id*, int gtid);
+extern void __kmpc_dispatch_init_4(id*, int, enum sched, int, int, int, int);
+extern void __kmpc_dispatch_init_8(id*, int, enum sched, i64, i64, i64, i64);
+extern int __kmpc_dispatch_next_4(id*, int, void*, void*, void*, void*);
+extern int __kmpc_dispatch_next_8(id*, int, void*, void*, void*, void*);
+// End of definitions copied from OpenMP RTL.
+// ---------------------------------------------------------------------------
+static id loc = {0, 2, 0, 0, ";file;func;0;0;;"};
+
+// ---------------------------------------------------------------------------
+int run_loop_64(i64 loop_lb, i64 loop_ub, i64 loop_st, int loop_chunk) {
+  int err = 0;
+  static int volatile loop_sync = 0;
+  i64 lb;   // Chunk lower bound
+  i64 ub;   // Chunk upper bound
+  i64 st;   // Chunk stride
+  int rc;
+  int tid = omp_get_thread_num();
+  int gtid = tid;
+  int last;
+#if DEBUG
+  printf("run_loop_<%d>(lb=%d, ub=%d, st=%d, ch=%d)\n",
+    (int)sizeof(i64), gtid, tid,
+    (int)loop_lb, (int)loop_ub, (int)loop_st, loop_chunk);
+#endif
+  // Don't test degenerate cases that should have been discovered by codegen
+  if (loop_st == 0)
+    return 0;
+  if (loop_st > 0 ? loop_lb > loop_ub : loop_lb < loop_ub)
+    return 0;
+
+  __kmpc_dispatch_init_8(&loc, gtid, kmp_sch_guided_simd,
+                         loop_lb, loop_ub, loop_st, loop_chunk);
+  if (tid == 0) {
+    // Let the master thread handle the chunks alone
+    int chunk;      // No of current chunk
+    i64 next_lb;    // Lower bound of the next chunk
+    i64 last_ub;    // Upper bound of the last processed chunk
+    u64 cur;        // Number of interations in  current chunk
+    u64 max;        // Max allowed iterations for current chunk
+    int undersized = 0;
+
+    chunk = 0;
+    next_lb = loop_lb;
+    max = (loop_ub - loop_lb) / loop_st + 1;
+    // The first chunk can consume all iterations
+    while (__kmpc_dispatch_next_8(&loc, gtid, &last, &lb, &ub, &st)) {
+      ++ chunk;
+#if DEBUG
+      printf("chunk=%d, lb=%d, ub=%d\n", chunk, (int)lb, (int)ub);
+#endif
+      // Check if previous chunk (it is not the final chunk) is undersized
+      if (undersized) {
+        printf("Error with chunk %d\n", chunk);
+        err++;
+      }
+      // Check lower and upper bounds
+      if (lb != next_lb) {
+        printf("Error with lb %d, %d, ch %d\n", (int)lb, (int)next_lb, chunk);
+        err++;
+      }
+      if (loop_st > 0) {
+        if (!(ub <= loop_ub)) {
+          printf("Error with ub %d, %d, ch %d\n", (int)ub, (int)loop_ub, chunk);
+          err++;
+        }
+        if (!(lb <= ub)) {
+          printf("Error with bounds %d, %d, %d\n", (int)lb, (int)ub, chunk);
+          err++;
+        }
+      } else {
+        if (!(ub >= loop_ub)) {
+          printf("Error with ub %d, %d, %d\n", (int)ub, (int)loop_ub, chunk);
+          err++;
+        }
+        if (!(lb >= ub)) {
+          printf("Error with bounds %d, %d, %d\n", (int)lb, (int)ub, chunk);
+          err++;
+        }
+      }; // if
+      // Stride should not change
+      if (!(st == loop_st)) {
+        printf("Error with st %d, %d, ch %d\n", (int)st, (int)loop_st, chunk);
+        err++;
+      }
+      cur = (ub - lb) / loop_st + 1;
+      // Guided scheduling uses FP computations, so current chunk may
+      // be a bit bigger (+1) than allowed maximum
+      if (!(cur <= max + 1)) {
+        printf("Error with iter %d, %d\n", cur, max);
+        err++;
+      }
+      // Update maximum for the next chunk
+      if (cur < max)
+        max = cur;
+      next_lb = ub + loop_st;
+      last_ub = ub;
+      undersized = (cur < loop_chunk);
+    }; // while
+    // Must have at least one chunk
+    if (!(chunk > 0)) {
+      printf("Error with chunk %d\n", chunk);
+      err++;
+    }
+    // Must have the right last iteration index
+    if (loop_st > 0) {
+      if (!(last_ub <= loop_ub)) {
+        printf("Error with last1 %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_ub, chunk);
+        err++;
+      }
+      if (!(last_ub + loop_st > loop_ub)) {
+        printf("Error with last2 %d, %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_st, (int)loop_ub, chunk);
+        err++;
+      }
+    } else {
+      if (!(last_ub >= loop_ub)) {
+        printf("Error with last1 %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_ub, chunk);
+        err++;
+      }
+      if (!(last_ub + loop_st < loop_ub)) {
+        printf("Error with last2 %d, %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_st, (int)loop_ub, chunk);
+        err++;
+      }
+    }; // if
+    // Let non-master threads go
+    loop_sync = 1;
+  } else {
+    int i;
+    // Workers wait for master thread to finish, then call __kmpc_dispatch_next
+    for (i = 0; i < 1000000; ++ i) {
+      if (loop_sync != 0) {
+        break;
+      }; // if
+    }; // for i
+    while (loop_sync == 0) {
+      delay();
+    }; // while
+    // At this moment we do not have any more chunks -- all the chunks already
+    // processed by master thread
+    rc = __kmpc_dispatch_next_8(&loc, gtid, &last, &lb, &ub, &st);
+    if (rc) {
+      printf("Error return value\n");
+      err++;
+    }
+  }; // if
+
+  __kmpc_barrier(&loc, gtid);
+  if (tid == 0) {
+      loop_sync = 0;    // Restore original state
+#if DEBUG
+      printf("run_loop_64(): at the end\n");
+#endif
+  }; // if
+  __kmpc_barrier(&loc, gtid);
+  return err;
+} // run_loop
+
+// ---------------------------------------------------------------------------
+int run_loop_32(int loop_lb, int loop_ub, int loop_st, int loop_chunk) {
+  int err = 0;
+  static int volatile loop_sync = 0;
+  int lb;   // Chunk lower bound
+  int ub;   // Chunk upper bound
+  int st;   // Chunk stride
+  int rc;
+  int tid = omp_get_thread_num();
+  int gtid = tid;
+  int last;
+#if DEBUG
+  printf("run_loop_<%d>(lb=%d, ub=%d, st=%d, ch=%d)\n",
+    (int)sizeof(int), gtid, tid,
+    (int)loop_lb, (int)loop_ub, (int)loop_st, loop_chunk);
+#endif
+  // Don't test degenerate cases that should have been discovered by codegen
+  if (loop_st == 0)
+    return 0;
+  if (loop_st > 0 ? loop_lb > loop_ub : loop_lb < loop_ub)
+    return 0;
+
+  __kmpc_dispatch_init_4(&loc, gtid, kmp_sch_guided_simd,
+                         loop_lb, loop_ub, loop_st, loop_chunk);
+  if (tid == 0) {
+    // Let the master thread handle the chunks alone
+    int chunk;      // No of current chunk
+    int next_lb;    // Lower bound of the next chunk
+    int last_ub;    // Upper bound of the last processed chunk
+    u64 cur;        // Number of interations in  current chunk
+    u64 max;        // Max allowed iterations for current chunk
+    int undersized = 0;
+
+    chunk = 0;
+    next_lb = loop_lb;
+    max = (loop_ub - loop_lb) / loop_st + 1;
+    // The first chunk can consume all iterations
+    while (__kmpc_dispatch_next_4(&loc, gtid, &last, &lb, &ub, &st)) {
+      ++ chunk;
+#if DEBUG
+      printf("chunk=%d, lb=%d, ub=%d\n", chunk, (int)lb, (int)ub);
+#endif
+      // Check if previous chunk (it is not the final chunk) is undersized
+      if (undersized) {
+        printf("Error with chunk %d\n", chunk);
+        err++;
+      }
+      // Check lower and upper bounds
+      if (lb != next_lb) {
+        printf("Error with lb %d, %d, ch %d\n", (int)lb, (int)next_lb, chunk);
+        err++;
+      }
+      if (loop_st > 0) {
+        if (!(ub <= loop_ub)) {
+          printf("Error with ub %d, %d, ch %d\n", (int)ub, (int)loop_ub, chunk);
+          err++;
+        }
+        if (!(lb <= ub)) {
+          printf("Error with bounds %d, %d, %d\n", (int)lb, (int)ub, chunk);
+          err++;
+        }
+      } else {
+        if (!(ub >= loop_ub)) {
+          printf("Error with ub %d, %d, %d\n", (int)ub, (int)loop_ub, chunk);
+          err++;
+        }
+        if (!(lb >= ub)) {
+          printf("Error with bounds %d, %d, %d\n", (int)lb, (int)ub, chunk);
+          err++;
+        }
+      }; // if
+      // Stride should not change
+      if (!(st == loop_st)) {
+        printf("Error with st %d, %d, ch %d\n", (int)st, (int)loop_st, chunk);
+        err++;
+      }
+      cur = (ub - lb) / loop_st + 1;
+      // Guided scheduling uses FP computations, so current chunk may
+      // be a bit bigger (+1) than allowed maximum
+      if (!(cur <= max + 1)) {
+        printf("Error with iter %d, %d\n", cur, max);
+        err++;
+      }
+      // Update maximum for the next chunk
+      if (cur < max)
+        max = cur;
+      next_lb = ub + loop_st;
+      last_ub = ub;
+      undersized = (cur < loop_chunk);
+    }; // while
+    // Must have at least one chunk
+    if (!(chunk > 0)) {
+      printf("Error with chunk %d\n", chunk);
+      err++;
+    }
+    // Must have the right last iteration index
+    if (loop_st > 0) {
+      if (!(last_ub <= loop_ub)) {
+        printf("Error with last1 %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_ub, chunk);
+        err++;
+      }
+      if (!(last_ub + loop_st > loop_ub)) {
+        printf("Error with last2 %d, %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_st, (int)loop_ub, chunk);
+        err++;
+      }
+    } else {
+      if (!(last_ub >= loop_ub)) {
+        printf("Error with last1 %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_ub, chunk);
+        err++;
+      }
+      if (!(last_ub + loop_st < loop_ub)) {
+        printf("Error with last2 %d, %d, %d, ch %d\n",
+               (int)last_ub, (int)loop_st, (int)loop_ub, chunk);
+        err++;
+      }
+    }; // if
+    // Let non-master threads go
+    loop_sync = 1;
+  } else {
+    int i;
+    // Workers wait for master thread to finish, then call __kmpc_dispatch_next
+    for (i = 0; i < 1000000; ++ i) {
+      if (loop_sync != 0) {
+        break;
+      }; // if
+    }; // for i
+    while (loop_sync == 0) {
+      delay();
+    }; // while
+    // At this moment we do not have any more chunks -- all the chunks already
+    // processed by the master thread
+    rc = __kmpc_dispatch_next_4(&loc, gtid, &last, &lb, &ub, &st);
+    if (rc) {
+      printf("Error return value\n");
+      err++;
+    }
+  }; // if
+
+  __kmpc_barrier(&loc, gtid);
+  if (tid == 0) {
+      loop_sync = 0;    // Restore original state
+#if DEBUG
+      printf("run_loop<>(): at the end\n");
+#endif
+  }; // if
+  __kmpc_barrier(&loc, gtid);
+  return err;
+} // run_loop
+
+// ---------------------------------------------------------------------------
+int run_64(int num_th)
+{
+ int err = 0;
+#pragma omp parallel num_threads(num_th)
+ {
+  int chunk;
+  i64 st, lb, ub;
+  for (chunk = SIMD_LEN; chunk <= 3*SIMD_LEN; chunk += SIMD_LEN) {
+    for (st = 1; st <= 3; ++ st) {
+      for (lb = -3 * num_th * st; lb <= 3 * num_th * st; ++ lb) {
+        for (ub = lb; ub < lb + num_th * (chunk+1) * st; ++ ub) {
+          err += run_loop_64(lb, ub,  st, chunk);
+          err += run_loop_64(ub, lb, -st, chunk);
+        }; // for ub
+      }; // for lb
+    }; // for st
+  }; // for chunk
+ }
+ return err;
+} // run_all
+
+int run_32(int num_th)
+{
+ int err = 0;
+#pragma omp parallel num_threads(num_th)
+ {
+  int chunk, st, lb, ub;
+  for (chunk = SIMD_LEN; chunk <= 3*SIMD_LEN; chunk += SIMD_LEN) {
+    for (st = 1; st <= 3; ++ st) {
+      for (lb = -3 * num_th * st; lb <= 3 * num_th * st; ++ lb) {
+        for (ub = lb; ub < lb + num_th * (chunk+1) * st; ++ ub) {
+          err += run_loop_32(lb, ub,  st, chunk);
+          err += run_loop_32(ub, lb, -st, chunk);
+        }; // for ub
+      }; // for lb
+    }; // for st
+  }; // for chunk
+ }
+ return err;
+} // run_all
+
+// ---------------------------------------------------------------------------
+int main()
+{
+  int n, err = 0;
+  for (n = 1; n <= 4; ++ n) {
+    err += run_32(n);
+    err += run_64(n);
+  }; // for n
+  if (err)
+    printf("failed with %d errors\n", err);
+  else
+    printf("passed\n");
+  return err;
+}