dotest.py - debug feature that helps find dosep races

[vharron]

Summary

dosep.py starts lots and lots of dotest instances.

This option helps you find if two (or more) dotest instances are using
the same directory at the same time.

Enable it to cause test failures and stderr messages if dotest
instances try to run in the same directory simultaneously.

It is disabled by default because it litters the test directories with
".dirlock" files

[labath]

LGTM.
I have been also worried by this problem, but manual code inspection assured me that this could not happen. It will be interesting to see if this finds anything, as some of the rare failures we're seeing do seem to hint at the possibility of something like this happening.

test/lldbtest.py

57

How will you set this to true? Shouldn't we add a command line option, environment variable or something?

[vharron]

I'll add a command line parameter in a followup CL (when we suspect this is a problem again)

test/lldbtest.py

 Ran 72 tests in 135.468s
 
 OK
 $ 
 """
 
 import abc
 import glob
+import lock
 import os, sys, traceback
 import os.path
 import re
 import signal
 from subprocess import *
 import StringIO
 import time
 import types
 import unittest2
 import lldb
 import lldbtest_config
 import lldbutil
 from _pyio import __metaclass__
 
+# dosep.py starts lots and lots of dotest instances
+# This option helps you find if two (or more) dotest instances are using the same
+# directory at the same time
+# Enable it to cause test failures and stderr messages if dotest instances try to run in
+# the same directory simultaneously
+# it is disabled by default because it litters the test directories with ".dirlock" files
+debug_confirm_directory_exclusivity = False
+
 # See also dotest.parseOptionsAndInitTestdirs(), where the environment variables
 # LLDB_COMMAND_TRACE and LLDB_DO_CLEANUP are set from '-t' and '-r dir' options.
 
 # By default, traceAlways is False.
 if "LLDB_COMMAND_TRACE" in os.environ and os.environ["LLDB_COMMAND_TRACE"]=="YES":
     traceAlways = True
 else:
     traceAlways = False
def TraceOn(self):
         return traceAlways
     
     @classmethod
     def setUpClass(cls):
         """
         Python unittest framework class setup fixture.
         Do current directory manipulation.
         """
-
         # Fail fast if 'mydir' attribute is not overridden.
         if not cls.mydir or len(cls.mydir) == 0:
             raise Exception("Subclasses must override the 'mydir' attribute.")
 
         # Save old working directory.
         cls.oldcwd = os.getcwd()
 
         # Change current working directory if ${LLDB_TEST} is defined.
         # See also dotest.py which sets up ${LLDB_TEST}.
         if ("LLDB_TEST" in os.environ):
+            full_dir = os.path.join(os.environ["LLDB_TEST"], cls.mydir)
             if traceAlways:
-                print >> sys.stderr, "Change dir to:", os.path.join(os.environ["LLDB_TEST"], cls.mydir)
+                print >> sys.stderr, "Change dir to:", full_dir
             os.chdir(os.path.join(os.environ["LLDB_TEST"], cls.mydir))
 
+        if debug_confirm_directory_exclusivity:
+            cls.dir_lock = lock.Lock(os.path.join(full_dir, ".dirlock"))
+            try:
+                cls.dir_lock.try_acquire()
+                # write the class that owns the lock into the lock file
+                cls.dir_lock.handle.write(cls.__name__)
+            except IOError as ioerror:
+                # nothing else should have this directory lock
+                # wait here until we get a lock
+                cls.dir_lock.acquire()
+                # read the previous owner from the lock file
+                lock_id = cls.dir_lock.handle.read()
+                print >> sys.stderr, "LOCK ERROR: {} wants to lock '{}' but it is already locked by '{}'".format(cls.__name__, full_dir, lock_id)
+                raise ioerror
+
         # Set platform context.
         if platformIsDarwin():
             cls.platformContext = _PlatformContext('DYLD_LIBRARY_PATH', 'lib', 'dylib')
         elif getPlatform() == "linux" or getPlatform() == "freebsd":
             cls.platformContext = _PlatformContext('LD_LIBRARY_PATH', 'lib', 'so')
         else:
             cls.platformContext = None
 
def tearDownClass(cls):
                 if traceAlways:
                     print >> sys.stderr, "Call class-specific cleanup function for class:", cls
                 try:
                     cls.classCleanup()
                 except:
                     exc_type, exc_value, exc_tb = sys.exc_info()
                     traceback.print_exception(exc_type, exc_value, exc_tb)
 
+        if debug_confirm_directory_exclusivity:
+            cls.dir_lock.release()
+            del cls.dir_lock
+
         # Restore old working directory.
         if traceAlways:
             print >> sys.stderr, "Restore dir to:", cls.oldcwd
         os.chdir(cls.oldcwd)
 
     @classmethod
     def skipLongRunningTest(cls):
         """

test/lock.py

 """
 Interprocess mutex based on file locks
 """
 
 import fcntl
 import os
 
 class Lock:
 
     def __init__(self, filename):
         self.filename = filename
         # This will create it if it does not exist already
-        self.handle = open(filename, 'w')
+        unbuffered = 0
+        self.handle = open(filename, 'a+', unbuffered)
 
-    # Bitwise OR fcntl.LOCK_NB if you need a non-blocking lock
     def acquire(self):
         fcntl.flock(self.handle, fcntl.LOCK_EX)
 
+    # will throw IOError if unavailable
+    def try_acquire(self):
+        fcntl.flock(self.handle, fcntl.LOCK_NB | fcntl.LOCK_EX)
+
     def release(self):
         fcntl.flock(self.handle, fcntl.LOCK_UN)
 
     def __del__(self):
         self.handle.close()