[OpenMP] Fix /tmp approach, and add environment variable method as third fallback during library registration

[tlwilmar]

Summary

The /tmp fallback for /dev/shm did not write to a fixed filename, so multiple instances of the runtime would not be able to detect each other. Now, we create the /tmp file in much the same way as the /dev/shm file was created, since mkstemp approach would not work to create a file that other instances of the runtime would detect. Also, add the environment variable method as a third fallback to /dev/shm and /tmp for library registration, as some systems do not have either. Also, add ability to fallback to a subsequent method should a failure occur during any part of the registration process. When unregistering, it is assumed that the method chosen during registration should work, so errors at that point are ignored. This also avoids a problem with multiple threads trying to unregister the library.

[jlpeyton]

LGTM

openmp/runtime/src/kmp.h

 extern void __kmp_exit_thread(int exit_status);
 extern void __kmp_abort(char const *format, ...);
 extern void __kmp_abort_thread(void);
 KMP_NORETURN extern void __kmp_abort_process(void);
 extern void __kmp_warn(char const *format, ...);
 
 extern void __kmp_set_num_threads(int new_nth, int gtid);
 
+extern bool __kmp_detect_shm();
+extern bool __kmp_detect_tmp();
+
 // Returns current thread (pointer to kmp_info_t). Current thread *must* be
 // registered.
 static inline kmp_info_t *__kmp_entry_thread() {
   int gtid = __kmp_entry_gtid();
 
   return __kmp_threads[gtid];
 }
 

openmp/runtime/src/kmp_runtime.cpp

#if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB
   return __kmp_str_format("__KMP_REGISTERED_LIB_%d_%d", (int)getpid(),
                           (int)getuid());
 #else
   return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
 #endif
 } // __kmp_reg_status_get
 
 #if defined(KMP_USE_SHM)
+bool __kmp_shm_available = false;
+bool __kmp_tmp_available = false;
 // If /dev/shm is not accessible, we will create a temporary file under /tmp.
 char *temp_reg_status_file_name = nullptr;
 #endif
 
 void __kmp_register_library_startup(void) {
 
   char *name = __kmp_reg_status_name(); // Name of the environment variable.
   int done = 0;
#endif
   KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,
                 __kmp_registration_str));
 
   while (!done) {
 
     char *value = NULL; // Actual value of the environment variable.
 
 #if defined(KMP_USE_SHM)
-    char *shm_name = __kmp_str_format("/%s", name);
+    char *shm_name = nullptr;
+    char *data1 = nullptr;
+    __kmp_shm_available = __kmp_detect_shm();
+    if (__kmp_shm_available) {
+      int fd1 = -1;
+      shm_name = __kmp_str_format("/%s", name);
       int shm_preexist = 0;
-    char *data1;
-    int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
+      fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
       if ((fd1 == -1) && (errno == EEXIST)) {
         // file didn't open because it already exists.
         // try opening existing file
         fd1 = shm_open(shm_name, O_RDWR, 0666);
         if (fd1 == -1) { // file didn't open
-        // error out here
-        __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM"), KMP_ERR(0),
-                    __kmp_msg_null);
-      } else {
-        // able to open existing file
+          KMP_WARNING(FunctionError, "Can't open SHM");
+          __kmp_shm_available = false;
+        } else { // able to open existing file
           shm_preexist = 1;
         }
-    } else if (fd1 == -1) {
-      // SHM didn't open; it was due to error other than already exists. Try to
-      // create a temp file under /tmp.
-      // TODO: /tmp might not always be the temporary directory. For now we will
-      // not consider TMPDIR. If /tmp is not accessible, we simply error out.
-      char *temp_file_name = __kmp_str_format("/tmp/%sXXXXXX", name);
-      fd1 = mkstemp(temp_file_name);
-      if (fd1 == -1) {
-        // error out here.
-        __kmp_fatal(KMP_MSG(FunctionError, "Can't open TEMP"), KMP_ERR(errno),
-                    __kmp_msg_null);
       }
-      temp_reg_status_file_name = temp_file_name;
+      if (__kmp_shm_available && shm_preexist == 0) { // SHM created, set size
+        if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;
+          KMP_WARNING(FunctionError, "Can't set size of SHM");
+          __kmp_shm_available = false;
         }
-    if (shm_preexist == 0) {
-      // we created SHM now set size
-      if (ftruncate(fd1, SHM_SIZE) == -1) {
-        // error occured setting size;
-        __kmp_fatal(KMP_MSG(FunctionError, "Can't set size of SHM"),
-                    KMP_ERR(errno), __kmp_msg_null);
       }
+      if (__kmp_shm_available) { // SHM exists, now map it
+        data1 = (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
+                             fd1, 0);
+        if (data1 == MAP_FAILED) { // failed to map shared memory
+          KMP_WARNING(FunctionError, "Can't map SHM");
+          __kmp_shm_available = false;
         }
-    data1 =
-        (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
-    if (data1 == MAP_FAILED) {
-      // failed to map shared memory
-      __kmp_fatal(KMP_MSG(FunctionError, "Can't map SHM"), KMP_ERR(errno),
-                  __kmp_msg_null);
       }
+      if (__kmp_shm_available) { // SHM mapped
         if (shm_preexist == 0) { // set data to SHM, set value
           KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
         }
         // Read value from either what we just wrote or existing file.
         value = __kmp_str_format("%s", data1); // read value from SHM
         munmap(data1, SHM_SIZE);
+      }
+      if (fd1 != -1)
+        close(fd1);
+    }
+    if (!__kmp_shm_available)
+      __kmp_tmp_available = __kmp_detect_tmp();
+    if (!__kmp_shm_available && __kmp_tmp_available) {
+      // SHM failed to work due to an error other than that the file already
+      // exists. Try to create a temp file under /tmp.
+      // If /tmp isn't accessible, fall back to using environment variable.
+      // TODO: /tmp might not always be the temporary directory. For now we will
+      // not consider TMPDIR.
+      int fd1 = -1;
+      temp_reg_status_file_name = __kmp_str_format("/tmp/%s", name);
+      int tmp_preexist = 0;
+      fd1 = open(temp_reg_status_file_name, O_CREAT | O_EXCL | O_RDWR, 0666);
+      if ((fd1 == -1) && (errno == EEXIST)) {
+        // file didn't open because it already exists.
+        // try opening existing file
+        fd1 = open(temp_reg_status_file_name, O_RDWR, 0666);
+        if (fd1 == -1) { // file didn't open if (fd1 == -1) {
+          KMP_WARNING(FunctionError, "Can't open TEMP");
+          __kmp_tmp_available = false;
+        } else {
+          tmp_preexist = 1;
+        }
+      }
+      if (__kmp_tmp_available && tmp_preexist == 0) {
+        // we created /tmp file now set size
+        if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;
+          KMP_WARNING(FunctionError, "Can't set size of /tmp file");
+          __kmp_tmp_available = false;
+        }
+      }
+      if (__kmp_tmp_available) {
+        data1 = (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
+                             fd1, 0);
+        if (data1 == MAP_FAILED) { // failed to map /tmp
+          KMP_WARNING(FunctionError, "Can't map /tmp");
+          __kmp_tmp_available = false;
+        }
+      }
+      if (__kmp_tmp_available) {
+        if (tmp_preexist == 0) { // set data to TMP, set value
+          KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
+        }
+        // Read value from either what we just wrote or existing file.
+        value = __kmp_str_format("%s", data1); // read value from SHM
+        munmap(data1, SHM_SIZE);
+      }
+      if (fd1 != -1)
         close(fd1);
+    }
+    if (!__kmp_shm_available && !__kmp_tmp_available) {
+      // no /dev/shm and no /tmp -- fall back to environment variable
+      // Set environment variable, but do not overwrite if it exists.
+      __kmp_env_set(name, __kmp_registration_str, 0);
+      // read value to see if it got set
+      value = __kmp_env_get(name);
+    }
 #else // Windows and unix with static library
-    // Set environment variable, but do not overwrite if it is exist.
+    // Set environment variable, but do not overwrite if it exists.
     __kmp_env_set(name, __kmp_registration_str, 0);
     // read value to see if it got set
     value = __kmp_env_get(name);
 #endif
 
     if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
       done = 1; // Ok, environment variable set successfully, exit the loop.
     } else {
if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
         }
         KMP_INTERNAL_FREE(duplicate_ok);
         __kmp_duplicate_library_ok = 1;
         done = 1; // Exit the loop.
       } break;
       case 2: { // Neighbor is dead.
 
 #if defined(KMP_USE_SHM)
-        // close shared memory.
+        if (__kmp_shm_available) { // close shared memory.
           shm_unlink(shm_name); // this removes file in /dev/shm
+        } else if (__kmp_tmp_available) {
+          unlink(temp_reg_status_file_name); // this removes the temp file
+        } else {
+          // Clear the variable and try to register library again.
+          __kmp_env_unset(name);
+        }
 #else
         // Clear the variable and try to register library again.
         __kmp_env_unset(name);
 #endif
       } break;
       default: {
         KMP_DEBUG_ASSERT(0);
       } break;
       }
     }
     KMP_INTERNAL_FREE((void *)value);
 #if defined(KMP_USE_SHM)
+    if (shm_name)
       KMP_INTERNAL_FREE((void *)shm_name);
 #endif
   } // while
   KMP_INTERNAL_FREE((void *)name);
 
 } // func __kmp_register_library_startup
 
 void __kmp_unregister_library(void) {
 
   char *name = __kmp_reg_status_name();
   char *value = NULL;
 
 #if defined(KMP_USE_SHM)
-  bool use_shm = true;
-  char *shm_name = __kmp_str_format("/%s", name);
-  int fd1 = shm_open(shm_name, O_RDONLY, 0666);
-  if (fd1 == -1) {
-    // File did not open. Try the temporary file.
-    use_shm = false;
-    KMP_DEBUG_ASSERT(temp_reg_status_file_name);
-    fd1 = open(temp_reg_status_file_name, O_RDONLY);
-    if (fd1 == -1) {
-      // give it up now.
-      return;
+  char *shm_name = nullptr;
+  int fd1;
+  if (__kmp_shm_available) {
+    shm_name = __kmp_str_format("/%s", name);
+    fd1 = shm_open(shm_name, O_RDONLY, 0666);
+    if (fd1 != -1) { // File opened successfully
+      char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
+      if (data1 != MAP_FAILED) {
+        value = __kmp_str_format("%s", data1); // read value from SHM
+        munmap(data1, SHM_SIZE);
       }
+      close(fd1);
     }
+  } else if (__kmp_tmp_available) { // try /tmp
+    fd1 = open(temp_reg_status_file_name, O_RDONLY);
+    if (fd1 != -1) { // File opened successfully
       char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
       if (data1 != MAP_FAILED) {
-    value = __kmp_str_format("%s", data1); // read value from SHM
+        value = __kmp_str_format("%s", data1); // read value from /tmp
         munmap(data1, SHM_SIZE);
       }
       close(fd1);
+    }
+  } else { // fall back to envirable
+    value = __kmp_env_get(name);
+  }
 #else
   value = __kmp_env_get(name);
 #endif
 
   KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
   KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
   if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
 //  Ok, this is our variable. Delete it.
 #if defined(KMP_USE_SHM)
-    if (use_shm) {
+    if (__kmp_shm_available) {
       shm_unlink(shm_name); // this removes file in /dev/shm
-    } else {
-      KMP_DEBUG_ASSERT(temp_reg_status_file_name);
+    } else if (__kmp_tmp_available) {
       unlink(temp_reg_status_file_name); // this removes the temp file
+    } else {
+      __kmp_env_unset(name);
     }
 #else
     __kmp_env_unset(name);
 #endif
   }
 
 #if defined(KMP_USE_SHM)
+  if (shm_name)
     KMP_INTERNAL_FREE(shm_name);
-  if (!use_shm) {
-    KMP_DEBUG_ASSERT(temp_reg_status_file_name);
+  if (temp_reg_status_file_name)
     KMP_INTERNAL_FREE(temp_reg_status_file_name);
-  }
 #endif
 
   KMP_INTERNAL_FREE(__kmp_registration_str);
   KMP_INTERNAL_FREE(value);
   KMP_INTERNAL_FREE(name);
 
   __kmp_registration_flag = 0;
   __kmp_registration_str = NULL;

openmp/runtime/src/z_Linux_util.cpp

void __kmp_hidden_helper_threads_deinitz_wait() {
   KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
 }
 
 void __kmp_hidden_helper_threads_deinitz_release() {
   KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
 }
 #endif // KMP_OS_LINUX
 
+bool __kmp_detect_shm() {
+  DIR *dir = opendir("/dev/shm");
+  if (dir) { // /dev/shm exists
+    closedir(dir);
+    return true;
+  } else if (ENOENT == errno) { // /dev/shm does not exist
+    return false;
+  } else { // opendir() failed
+    return false;
+  }
+}
+
+bool __kmp_detect_tmp() {
+  DIR *dir = opendir("/tmp");
+  if (dir) { // /tmp exists
+    closedir(dir);
+    return true;
+  } else if (ENOENT == errno) { // /tmp does not exist
+    return false;
+  } else { // opendir() failed
+    return false;
+  }
+}
+
 // end of file //