Use kevent queue for detecting file unlocking in LockFileManager

[PaulTaykalo]

Summary

LockFileManager is used for acquiring and watching .lock files
Current implementation that checks for lock release, is using exponentially growing timeouts.
In some cases, dependent module compilation time is relatively long (tens of seconds). In this case, timeouts could lead to the situation, when a process, watching for a lock release will just sleep for another ten of seconds after lock release.

Provided implementation is using kqueue for listening for file system changes process termination. This allows preventing unneded timeouts in the most of the cases.

[PaulTaykalo]

This is the somewhat an example of program that is using kevent queue. additional timeouts are added to test cases such as deletion of the file right after the queue was created but notlistened yet.
Kevent testing program (Pass event for first parameter)

#include <iostream>
#include <sys/event.h>
#include <unistd.h>
#include <fcntl.h>

// Class that employs RAII to save a file descriptor
class FileDescriptorKeeper {
    int FileDescriptor;

public:
    FileDescriptorKeeper(int Descriptor) { FileDescriptor = Descriptor; }

    ~FileDescriptorKeeper() {
        close(FileDescriptor);
        std::cout << "File successfully closed" << std::endl;
    }
};

bool ListenForAFileDeletion(char *path, int PID) {
    int LockFileDescriptor;
    int EventQueue;

    // Opening file for lock descriptor

    std::cout << "Opening file: " << path << std::endl;

    if ((LockFileDescriptor = open(path, O_RDONLY)) == -1) {
        std::cout << "[ERROR] File open error. The file was probably removed "
                  << path << std::endl;
        return false;
    }

    std::cout
            << "Will wait for 5 seconds to check if kqueu fails if file was deleted before event was added"
            << std::endl;

    sleep(5);
    FileDescriptorKeeper fileRAII = FileDescriptorKeeper(LockFileDescriptor);

    std::cout << "Creating queue: " << std::endl;

    if ((EventQueue = kqueue()) == -1) {
        std::cout << "[ERROR] Queue cannot be created " << errno << std::endl;
        return false;
    }

    FileDescriptorKeeper queueRAII = FileDescriptorKeeper(EventQueue);

    int ListeningEventsCount = 2;
    struct kevent ListeningEvents[2];
    struct kevent *FileRemovingEvent = &ListeningEvents[0];
    struct kevent *ProcessTerminationEvent = &ListeningEvents[1];

    EV_SET(FileRemovingEvent, LockFileDescriptor, EVFILT_VNODE,
           EV_ADD | EV_ENABLE | EV_ONESHOT | EV_CLEAR | EV_VANISHED |
           EV_DISPATCH2, NOTE_DELETE, 0, 0);

    EV_SET(ProcessTerminationEvent, PID, EVFILT_PROC,
           EV_ADD | EV_ENABLE | EV_ONESHOT, NOTE_EXIT, 0, 0);


    struct timespec timeout = {10, 0};
    struct kevent ReceivedEvent;

    int RegisteingResult =
            kevent(EventQueue, ListeningEvents, ListeningEventsCount, NULL, 0,
                   NULL);

    std::cout << "Registering events and start listening waiting for 5 sec"
              << RegisteingResult << std::endl;
    sleep(5);

    std::cout << "Listening event" << std::endl;

    int ListeningResult =
            kevent(EventQueue, NULL, 0, &ReceivedEvent, 1, &timeout);

    // File was successfully deleted
    if (ListeningResult > 0 && (ReceivedEvent.fflags & NOTE_DELETE &&
                            ReceivedEvent.filter == EVFILT_VNODE)) {
        std::cout << "[OK] File was deleted: " << path << std::endl;
        return true;
    }

    // Process owning the lock died
    if (ListeningResult > 0 && (ReceivedEvent.fflags & NOTE_EXIT &&
                                ReceivedEvent.filter == EVFILT_PROC)) {
        std::cout << "[OK] PID exited: " << path << std::endl;
        return true;
    }

    if (ListeningResult < 0) {
        std::cout << "[ERROR] Queue ended up with error" << errno << std::endl;
        return false;
    }

    if (ListeningResult == 0) {
        std::cout << "[OK] Queue has timed out" << std::endl;
        return false;
    }

    std::cout << "[??] Unexpected event received" << ReceivedEvent.fflags << ":"
              << ReceivedEvent.filter << std::endl;
    return false;
}

int main(int argc, char **argv) {
    if (argc < 2) {
        std::cout << "Please specify file to be deleted" << std::endl;
    }

    if (argc < 3) {
        std::cout << "Please specify pid to be listened to" << std::endl;
    }

    char *FileName = argv[1];
    int PID = atoi(argv[2]);

    ListenForAFileDeletion(FileName, PID);

    return 0;
}

llvm/include/llvm/Support/LockFileManager.h

private:
   std::string ErrorDiagMsg;
 
   LockFileManager(const LockFileManager &) = delete;
   LockFileManager &operator=(const LockFileManager &) = delete;
 
   static Optional<std::pair<std::string, int> >
   readLockFile(StringRef LockFileName);
 
+  bool waitForUnlockUsingSystemEvents(WaitForUnlockResult *);
+
   static bool processStillExecuting(StringRef Hostname, int PID);
+  static bool shouldListenForProcessState(StringRef HostID);
 
 public:
 
   LockFileManager(StringRef FileName);
   ~LockFileManager();
 
   /// Determine the state of the lock file.
   LockFileState getState() const;

llvm/lib/Support/LockFileManager.cpp

 #include <windows.h>
 #endif
 #if LLVM_ON_UNIX
 #include <unistd.h>
 #endif
 
 #if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && (__MAC_OS_X_VERSION_MIN_REQUIRED > 1050)
 #define USE_OSX_GETHOSTUUID 1
+#define USE_OSX_FILEWATCHER 1
 #else
 #define USE_OSX_GETHOSTUUID 0
+#define USE_OSX_FILEWATCHER 0
 #endif
 
 #if USE_OSX_GETHOSTUUID
 #include <uuid/uuid.h>
 #endif
 
+#if USE_OSX_FILEWATCHER
+#include <fcntl.h>
+#include <sys/event.h>
+#endif
 using namespace llvm;
 
 /// Attempt to read the lock file with the given name, if it exists.
 ///
 /// \param LockFileName The name of the lock file to read.
 ///
 /// \returns The process ID of the process that owns this lock file
 Optional<std::pair<std::string, int> >
#if LLVM_ON_UNIX && !defined(__ANDROID__)
   // Check whether the process is dead. If so, we're done.
   if (StoredHostID == HostID && getsid(PID) == -1 && errno == ESRCH)
     return false;
 #endif
 
   return true;
 }
 
+bool LockFileManager::shouldListenForProcessState(StringRef HostID) {
+#if LLVM_ON_UNIX && !defined(__ANDROID__)
+  SmallString<256> StoredHostID;
+  if (getHostID(StoredHostID))
+    // We cannot be sure that our host ID is the same as the specified one
+    // In this case we shouldn't listen for the PID status
+    return false;
+
+  // We shouldn't listen for Process changes on other hosts
+  if (StoredHostID != HostID)
+    return false;
+#endif
+
+  return false;
+}
+
+
 namespace {
 
 /// An RAII helper object ensure that the unique lock file is removed.
 ///
 /// Ensures that if there is an error or a signal before we finish acquiring the
 /// lock, the unique file will be removed. And if we successfully take the lock,
 /// the signal handler is left in place so that signals while the lock is held
 /// will remove the unique lock file. The caller should ensure there is a
LockFileManager::~LockFileManager() {
   // Since we own the lock, remove the lock file and our own unique lock file.
   sys::fs::remove(LockFileName);
   sys::fs::remove(UniqueLockFileName);
   // The unique file is now gone, so remove it from the signal handler. This
   // matches a sys::RemoveFileOnSignal() in LockFileManager().
   sys::DontRemoveFileOnSignal(UniqueLockFileName);
 }
 
+bool LockFileManager::waitForUnlockUsingSystemEvents(
+    LockFileManager::WaitForUnlockResult *Result) {
+#if !USE_OSX_FILEWATCHER
+  return false;
+#else
+
+  // Class that employs RAII to save a file descriptor
+  class FileDescriptorKeeper {
+    int FileDescriptor;
+
+  public:
+    FileDescriptorKeeper(int Descriptor) { FileDescriptor = Descriptor; }
+    ~FileDescriptorKeeper() { close(FileDescriptor); }
+  };
+
+  int EventQueue;
+  int LockFileDescriptor;
+
+  // Opening file for lock descriptor
+  if ((LockFileDescriptor = open(LockFileName.c_str(), O_RDONLY)) == -1)
+    return false;
+
+  FileDescriptorKeeper fileRAII = FileDescriptorKeeper(LockFileDescriptor);
+
+  // Creating a queue for listening
+  if ((EventQueue = kqueue()) == -1)
+    return false;
+
+  FileDescriptorKeeper queueRAII = FileDescriptorKeeper(EventQueue);
+
+  int ListeningEventsCount = 1;
+  struct kevent ListeningEvents[2];
+  struct kevent *FileRemovingEvent = &ListeningEvents[0];
+  struct kevent *ProcessTerminationEvent = &ListeningEvents[1];
+
+  // Listening for File Removing event
+  EV_SET(FileRemovingEvent, LockFileDescriptor, EVFILT_VNODE,
+         EV_ADD | EV_ENABLE | EV_ONESHOT, NOTE_DELETE, 0, 0);
+
+  // Additionally, We will listen for one more event if we're the running host
+  if (shouldListenForProcessState((*Owner).first)) {
+    EV_SET(ProcessTerminationEvent, (*Owner).second, EVFILT_PROC,
+           EV_ADD | EV_ENABLE | EV_ONESHOT, NOTE_EXIT, 0, 0);
+    ListeningEventsCount++;
+  }
+
+  int RegistrationResult =
+      kevent(EventQueue, ListeningEvents, ListeningEventsCount, nullptr, 0,
+          nullptr);
+
+  // We weren't able to register an event
+  if (RegistrationResult < 0)
+    return false;
+
+  // Theoretically there can be race condition, when file was deleted before
+  // we created an event queue. Now, when queue is created there's no way
+  // that we miss any event, but before that we need to check once more if
+  // file was deleted
+  if (sys::fs::access(LockFileName.c_str(), sys::fs::AccessMode::Exist) ==
+      errc::no_such_file_or_directory) {
+    // If the original file wasn't created, someone thought the lock was
+    // dead.
+    if (!sys::fs::exists(FileName)) {
+      *Result = Res_OwnerDied;
+      return true;
+    }
+    *Result = Res_Success;
+    return true;
+  }
+
+  // If the process owning the lock died without cleaning up, just bail
+  // out.
+  if (!processStillExecuting((*Owner).first, (*Owner).second)) {
+    *Result = Res_Success;
+    return true;
+  }
+
+  struct timespec timeout = {90, 0};
+  struct kevent ReceivedEvent;
+  int ListeningResult =
+      kevent(EventQueue, NULL, 0, &ReceivedEvent, 1, &timeout);
+
+  // File was successfully deleted
+  if (ListeningResult > 0 && (ReceivedEvent.fflags & NOTE_DELETE &&
+                          ReceivedEvent.filter == EVFILT_VNODE)) {
+    *Result = Res_Success;
+    return true;
+  }
+
+  // Process owning the lock died
+  if (ListeningResult > 0 && (ReceivedEvent.fflags & NOTE_EXIT &&
+                          ReceivedEvent.filter == EVFILT_PROC)) {
+    *Result = Res_Success;
+    return true;
+  }
+
+  // At this point it is either timeout or kevent ended up with some error
+  // Last checks before timing out
+
+  // Check if file was deleted
+  if (sys::fs::access(LockFileName.c_str(), sys::fs::AccessMode::Exist) ==
+      errc::no_such_file_or_directory) {
+    // If the original file wasn't created, someone thought the lock was
+    // dead.
+    if (!sys::fs::exists(FileName)) {
+      *Result = Res_OwnerDied;
+      return true;
+    }
+    *Result = Res_Success;
+    return true;
+  }
+
+  // If the process owning the lock died without cleaning up, just bail
+  // out.
+  if (!processStillExecuting((*Owner).first, (*Owner).second)) {
+    *Result = Res_Success;
+    return true;
+  }
+
+  // On timeout there'll be no events in the queue
+  if (ListeningResult == 0) {
+    *Result = Res_Timeout;
+    return true;
+  }
+
+  // For some reason we haven't received correct event from the file system
+  // And lock file is still present and owner process is still alive so we will
+  // probably fallback to the default implementation
+  return false;
+#endif
+}
+
 LockFileManager::WaitForUnlockResult LockFileManager::waitForUnlock() {
   if (getState() != LFS_Shared)
     return Res_Success;
 
+  WaitForUnlockResult SystemEventsResult;
+  if (waitForUnlockUsingSystemEvents(&SystemEventsResult)) {
+    return SystemEventsResult;
+  }
+
 #ifdef _WIN32
   unsigned long Interval = 1;
 #else
   struct timespec Interval;
   Interval.tv_sec = 0;
   Interval.tv_nsec = 1000000;
 #endif
   // Don't wait more than 40s per iteration. Total timeout for the file

 #endif
   } while (
 #ifdef _WIN32
            Interval < MaxSeconds * 1000
 #else
            Interval.tv_sec < (time_t)MaxSeconds
 #endif
            );
-
   // Give up.
   return Res_Timeout;
 }
 
 std::error_code LockFileManager::unsafeRemoveLockFile() {
   return sys::fs::remove(LockFileName);
 }