Index: cfe/trunk/include/clang/DirectoryWatcher/DirectoryWatcher.h =================================================================== --- cfe/trunk/include/clang/DirectoryWatcher/DirectoryWatcher.h +++ cfe/trunk/include/clang/DirectoryWatcher/DirectoryWatcher.h @@ -0,0 +1,123 @@ +//===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H +#define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include +#include +#include + +namespace clang { +/// Provides notifications for file changes in a directory. +/// +/// Invokes client-provided function on every filesystem event in the watched +/// directory. Initially the the watched directory is scanned and for every file +/// found, an event is synthesized as if the file was added. +/// +/// This is not a general purpose directory monitoring tool - list of +/// limitations follows. +/// +/// Only flat directories with no subdirectories are supported. In case +/// subdirectories are present the behavior is unspecified - events *might* be +/// passed to Receiver on macOS (due to FSEvents being used) while they +/// *probably* won't be passed on Linux (due to inotify being used). +/// +/// Known potential inconsistencies +/// - For files that are deleted befor the initial scan processed them, clients +/// might receive Removed notification without any prior Added notification. +/// - Multiple notifications might be produced when a file is added to the +/// watched directory during the initial scan. We are choosing the lesser evil +/// here as the only known alternative strategy would be to invalidate the +/// watcher instance and force user to create a new one whenever filesystem +/// event occurs during the initial scan but that would introduce continuous +/// restarting failure mode (watched directory is not always "owned" by the same +/// process that is consuming it). Since existing clients can handle duplicate +/// events well, we decided for simplicity. +/// +/// Notifications are provided only for changes done through local user-space +/// filesystem interface. Specifically, it's unspecified if notification would +/// be provided in case of a: +/// - a file mmap-ed and changed +/// - a file changed via remote (NFS) or virtual (/proc) FS access to monitored +/// directory +/// - another filesystem mounted to the watched directory +/// +/// No support for LLVM VFS. +/// +/// It is unspecified whether notifications for files being deleted are sent in +/// case the whole watched directory is sent. +/// +/// Directories containing "too many" files and/or receiving events "too +/// frequently" are not supported - if the initial scan can't be finished before +/// the watcher instance gets invalidated (see WatcherGotInvalidated) there's no +/// good error handling strategy - the only option for client is to destroy the +/// watcher, restart watching with new instance and hope it won't repeat. +class DirectoryWatcher { +public: + struct Event { + enum class EventKind { + Removed, + /// Content of a file was modified. + Modified, + /// The watched directory got deleted. + WatchedDirRemoved, + /// The DirectoryWatcher that originated this event is no longer valid and + /// its behavior is unspecified. + /// + /// The prime case is kernel signalling to OS-specific implementation of + /// DirectoryWatcher some resource limit being hit. + /// *Usually* kernel starts dropping or squashing events together after + /// that and so would DirectoryWatcher. This means that *some* events + /// might still be passed to Receiver but this behavior is unspecified. + /// + /// Another case is after the watched directory itself is deleted. + /// WatcherGotInvalidated will be received at least once during + /// DirectoryWatcher instance lifetime - when handling errors this is done + /// on best effort basis, when an instance is being destroyed then this is + /// guaranteed. + /// + /// The only proper response to this kind of event is to destruct the + /// originating DirectoryWatcher instance and create a new one. + WatcherGotInvalidated + }; + + EventKind Kind; + /// Filename that this event is related to or an empty string in + /// case this event is related to the watched directory itself. + std::string Filename; + + Event(EventKind Kind, llvm::StringRef Filename) + : Kind(Kind), Filename(Filename) {} + }; + + /// Returns nullptr if \param Path doesn't exist. + /// Returns nullptr if \param Path isn't a directory. + /// Returns nullptr if OS kernel API told us we can't start watching. In such + /// case it's unclear whether just retrying has any chance to succeeed. + static std::unique_ptr + create(llvm::StringRef Path, + std::function Events, + bool IsInitial)> + Receiver, + bool WaitForInitialSync); + + virtual ~DirectoryWatcher() = default; + DirectoryWatcher(const DirectoryWatcher &) = delete; + DirectoryWatcher &operator=(const DirectoryWatcher &) = delete; + DirectoryWatcher(DirectoryWatcher &&) = default; + +protected: + DirectoryWatcher() = default; +}; + +} // namespace clang + +#endif // LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H Index: cfe/trunk/lib/CMakeLists.txt =================================================================== --- cfe/trunk/lib/CMakeLists.txt +++ cfe/trunk/lib/CMakeLists.txt @@ -18,6 +18,7 @@ add_subdirectory(Frontend) add_subdirectory(FrontendTool) add_subdirectory(Tooling) +add_subdirectory(DirectoryWatcher) add_subdirectory(Index) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(StaticAnalyzer) Index: cfe/trunk/lib/DirectoryWatcher/CMakeLists.txt =================================================================== --- cfe/trunk/lib/DirectoryWatcher/CMakeLists.txt +++ cfe/trunk/lib/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,27 @@ +include(CheckIncludeFiles) + +set(LLVM_LINK_COMPONENTS support) + +set(DIRECTORY_WATCHER_SOURCES DirectoryScanner.cpp) +set(DIRECTORY_WATCHER_LINK_LIBS "") + +if(APPLE) + check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES) + if(HAVE_CORESERVICES) + list(APPEND DIRECTORY_WATCHER_SOURCES mac/DirectoryWatcher-mac.cpp) + set(DIRECTORY_WATCHER_LINK_LIBS "-framework CoreServices") + endif() +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + check_include_files("sys/inotify.h" HAVE_INOTIFY) + if(HAVE_INOTIFY) + list(APPEND DIRECTORY_WATCHER_SOURCES linux/DirectoryWatcher-linux.cpp) + find_package(Threads REQUIRED) + set(DIRECTORY_WATCHER_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) + endif() +endif() + +add_clang_library(clangDirectoryWatcher + ${DIRECTORY_WATCHER_SOURCES} + ) + +target_link_libraries(clangDirectoryWatcher PRIVATE ${DIRECTORY_WATCHER_LINK_LIBS}) Index: cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.h =================================================================== --- cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.h +++ cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.h @@ -0,0 +1,29 @@ +//===- DirectoryScanner.h - Utility functions for DirectoryWatcher --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include +#include + +namespace clang { + +/// Gets names (filenames) of items in directory at \p Path. +/// \returns empty vector if \p Path is not a directory, doesn't exist or can't +/// be read from. +std::vector scanDirectory(llvm::StringRef Path); + +/// Create event with EventKind::Added for every element in \p Scan. +std::vector +getAsFileEvents(const std::vector &Scan); + +/// Gets status of file (or directory) at \p Path. +/// \returns llvm::None if \p Path doesn't exist or can't get the status. +llvm::Optional getFileStatus(llvm::StringRef Path); + +} // namespace clang \ No newline at end of file Index: cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.cpp =================================================================== --- cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.cpp +++ cfe/trunk/lib/DirectoryWatcher/DirectoryScanner.cpp @@ -0,0 +1,54 @@ +//===- DirectoryScanner.cpp - Utility functions for DirectoryWatcher ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" + +#include "llvm/Support/Path.h" + +namespace clang { + +using namespace llvm; + +Optional getFileStatus(StringRef Path) { + sys::fs::file_status Status; + std::error_code EC = status(Path, Status); + if (EC) + return None; + return Status; +} + +std::vector scanDirectory(StringRef Path) { + using namespace llvm::sys; + std::vector Result; + + std::error_code EC; + for (auto It = fs::directory_iterator(Path, EC), + End = fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + auto status = getFileStatus(It->path()); + if (!status.hasValue()) + continue; + Result.emplace_back(sys::path::filename(It->path())); + } + + return Result; +} + +std::vector +getAsFileEvents(const std::vector &Scan) { + std::vector Events; + Events.reserve(Scan.size()); + + for (const auto &File : Scan) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::Modified, File}); + } + return Events; +} + +} // namespace clang \ No newline at end of file Index: cfe/trunk/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp =================================================================== --- cfe/trunk/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp +++ cfe/trunk/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp @@ -0,0 +1,345 @@ +//===- DirectoryWatcher-linux.cpp - Linux-platform directory watching -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/AlignOf.h" +#include "llvm/Support/Errno.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +using namespace llvm; +using namespace clang; + +/// Pipe for inter-thread synchronization - for epoll-ing on multiple +/// conditions. It is meant for uni-directional 1:1 signalling - specifically: +/// no multiple consumers, no data passing. Thread waiting for signal should +/// poll the FDRead. Signalling thread should call signal() which writes single +/// character to FDRead. +struct SemaphorePipe { + // Expects two file-descriptors opened as a pipe in the canonical POSIX + // order: pipefd[0] refers to the read end of the pipe. pipefd[1] refers to + // the write end of the pipe. + SemaphorePipe(int pipefd[2]) + : FDRead(pipefd[0]), FDWrite(pipefd[1]), OwnsFDs(true) {} + SemaphorePipe(const SemaphorePipe &) = delete; + void operator=(const SemaphorePipe &) = delete; + SemaphorePipe(SemaphorePipe &&other) + : FDRead(other.FDRead), FDWrite(other.FDWrite), + OwnsFDs(other.OwnsFDs) // Someone could have moved from the other + // instance before. + { + other.OwnsFDs = false; + }; + + void signal() { + ssize_t Result = llvm::sys::RetryAfterSignal(-1, write, FDWrite, "A", 1); + assert(Result != -1); + } + ~SemaphorePipe() { + if (OwnsFDs) { + close(FDWrite); + close(FDRead); + } + } + const int FDRead; + const int FDWrite; + bool OwnsFDs; + + static llvm::Optional create() { + int InotifyPollingStopperFDs[2]; + if (pipe2(InotifyPollingStopperFDs, O_CLOEXEC) == -1) + return llvm::None; + return SemaphorePipe(InotifyPollingStopperFDs); + } +}; + +/// Mutex-protected queue of Events. +class EventQueue { + std::mutex Mtx; + std::condition_variable NonEmpty; + std::queue Events; + +public: + void push_back(const DirectoryWatcher::Event::EventKind K, + StringRef Filename) { + { + std::unique_lock L(Mtx); + Events.emplace(K, Filename); + } + NonEmpty.notify_one(); + } + + // Blocks on caller thread and uses codition_variable to wait until there's an + // event to return. + DirectoryWatcher::Event pop_front_blocking() { + std::unique_lock L(Mtx); + while (true) { + // Since we might have missed all the prior notifications on NonEmpty we + // have to check the queue first (under lock). + if (!Events.empty()) { + DirectoryWatcher::Event Front = Events.front(); + Events.pop(); + return Front; + } + NonEmpty.wait(L, [this]() { return !Events.empty(); }); + } + } +}; + +class DirectoryWatcherLinux : public clang::DirectoryWatcher { +public: + DirectoryWatcherLinux( + llvm::StringRef WatchedDirPath, + std::function, bool)> Receiver, + bool WaitForInitialSync, int InotifyFD, int InotifyWD, + SemaphorePipe &&InotifyPollingStopSignal); + + ~DirectoryWatcherLinux() override { + StopWork(); + InotifyPollingThread.join(); + EventsReceivingThread.join(); + inotify_rm_watch(InotifyFD, InotifyWD); + llvm::sys::RetryAfterSignal(-1, close, InotifyFD); + } + +private: + const std::string WatchedDirPath; + // inotify file descriptor + int InotifyFD = -1; + // inotify watch descriptor + int InotifyWD = -1; + + EventQueue Queue; + + // Make sure lifetime of Receiver fully contains lifetime of + // EventsReceivingThread. + std::function, bool)> Receiver; + + // Consumes inotify events and pushes directory watcher events to the Queue. + void InotifyPollingLoop(); + std::thread InotifyPollingThread; + // Using pipe so we can epoll two file descriptors at once - inotify and + // stopping condition. + SemaphorePipe InotifyPollingStopSignal; + + // Does the initial scan of the directory - directly calling Receiver, + // bypassing the Queue. Both InitialScan and EventReceivingLoop use Receiver + // which isn't necessarily thread-safe. + void InitialScan(); + + // Processing events from the Queue. + // In case client doesn't want to do the initial scan synchronously + // (WaitForInitialSync=false in ctor) we do the initial scan at the beginning + // of this thread. + std::thread EventsReceivingThread; + // Push event of WatcherGotInvalidated kind to the Queue to stop the loop. + // Both InitialScan and EventReceivingLoop use Receiver which isn't + // necessarily thread-safe. + void EventReceivingLoop(); + + // Stops all the async work. Reentrant. + void StopWork() { + Queue.push_back(DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + ""); + InotifyPollingStopSignal.signal(); + } +}; + +void DirectoryWatcherLinux::InotifyPollingLoop() { + // We want to be able to read ~30 events at once even in the worst case + // (obscenely long filenames). + constexpr size_t EventBufferLength = + 30 * (sizeof(struct inotify_event) + NAME_MAX + 1); + // http://man7.org/linux/man-pages/man7/inotify.7.html + // Some systems cannot read integer variables if they are not + // properly aligned. On other systems, incorrect alignment may + // decrease performance. Hence, the buffer used for reading from + // the inotify file descriptor should have the same alignment as + // struct inotify_event. + + auto ManagedBuffer = + llvm::make_unique>(); + char *const Buf = ManagedBuffer->buffer; + + const int EpollFD = epoll_create1(EPOLL_CLOEXEC); + if (EpollFD == -1) { + StopWork(); + return; + } + auto EpollFDGuard = llvm::make_scope_exit([EpollFD]() { close(EpollFD); }); + + struct epoll_event EventSpec; + EventSpec.events = EPOLLIN; + EventSpec.data.fd = InotifyFD; + if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyFD, &EventSpec) == -1) { + StopWork(); + return; + } + + EventSpec.data.fd = InotifyPollingStopSignal.FDRead; + if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyPollingStopSignal.FDRead, + &EventSpec) == -1) { + StopWork(); + return; + } + + std::array EpollEventBuffer; + + while (true) { + const int EpollWaitResult = llvm::sys::RetryAfterSignal( + -1, epoll_wait, EpollFD, EpollEventBuffer.data(), + EpollEventBuffer.size(), /*timeout=*/-1 /*== infinity*/); + if (EpollWaitResult == -1) { + StopWork(); + return; + } + + // Multiple epoll_events can be received for a single file descriptor per + // epoll_wait call. + for (const auto &EpollEvent : EpollEventBuffer) { + if (EpollEvent.data.fd == InotifyPollingStopSignal.FDRead) { + StopWork(); + return; + } + } + + // epoll_wait() always return either error or >0 events. Since there was no + // event for stopping, it must be an inotify event ready for reading. + ssize_t NumRead = llvm::sys::RetryAfterSignal(-1, read, InotifyFD, Buf, + EventBufferLength); + for (char *P = Buf; P < Buf + NumRead;) { + if (P + sizeof(struct inotify_event) > Buf + NumRead) { + StopWork(); + llvm_unreachable("an incomplete inotify_event was read"); + return; + } + + struct inotify_event *Event = reinterpret_cast(P); + P += sizeof(struct inotify_event) + Event->len; + + if (Event->mask & (IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_DELETE) && + Event->len <= 0) { + StopWork(); + llvm_unreachable("expected a filename from inotify"); + return; + } + + if (Event->mask & (IN_CREATE | IN_MOVED_TO | IN_MODIFY)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Modified, + Event->name); + } else if (Event->mask & (IN_DELETE | IN_MOVED_FROM)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Removed, + Event->name); + } else if (Event->mask & (IN_DELETE_SELF | IN_MOVE_SELF)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::WatchedDirRemoved, + ""); + StopWork(); + return; + } else if (Event->mask & IN_IGNORED) { + StopWork(); + return; + } else { + StopWork(); + llvm_unreachable("Unknown event type."); + return; + } + } + } +} + +void DirectoryWatcherLinux::InitialScan() { + this->Receiver(getAsFileEvents(scanDirectory(WatchedDirPath)), + /*IsInitial=*/true); +} + +void DirectoryWatcherLinux::EventReceivingLoop() { + while (true) { + DirectoryWatcher::Event Event = this->Queue.pop_front_blocking(); + this->Receiver(Event, false); + if (Event.Kind == + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated) { + StopWork(); + return; + } + } +} + +DirectoryWatcherLinux::DirectoryWatcherLinux( + StringRef WatchedDirPath, + std::function, bool)> Receiver, + bool WaitForInitialSync, int InotifyFD, int InotifyWD, + SemaphorePipe &&InotifyPollingStopSignal) + : WatchedDirPath(WatchedDirPath), InotifyFD(InotifyFD), + InotifyWD(InotifyWD), Receiver(Receiver), + InotifyPollingStopSignal(std::move(InotifyPollingStopSignal)) { + + InotifyPollingThread = std::thread([this]() { InotifyPollingLoop(); }); + // We have no guarantees about thread safety of the Receiver which is being + // used in both InitialScan and EventReceivingLoop. We shouldn't run these + // only synchronously. + if (WaitForInitialSync) { + InitialScan(); + EventsReceivingThread = std::thread([this]() { EventReceivingLoop(); }); + } else { + EventsReceivingThread = std::thread([this]() { + // FIXME: We might want to terminate an async initial scan early in case + // of a failure in EventsReceivingThread. + InitialScan(); + EventReceivingLoop(); + }); + } +} + +} // namespace + +std::unique_ptr clang::DirectoryWatcher::create( + StringRef Path, + std::function, bool)> Receiver, + bool WaitForInitialSync) { + if (Path.empty()) + return nullptr; + + const int InotifyFD = inotify_init1(IN_CLOEXEC); + if (InotifyFD == -1) + return nullptr; + + const int InotifyWD = inotify_add_watch( + InotifyFD, Path.str().c_str(), + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_EXCL_UNLINK | IN_MODIFY | + IN_MOVED_FROM | IN_MOVE_SELF | IN_MOVED_TO | IN_ONLYDIR | IN_IGNORED); + if (InotifyWD == -1) + return nullptr; + + auto InotifyPollingStopper = SemaphorePipe::create(); + + if (!InotifyPollingStopper) + return nullptr; + + return llvm::make_unique( + Path, Receiver, WaitForInitialSync, InotifyFD, InotifyWD, + std::move(*InotifyPollingStopper)); +} \ No newline at end of file Index: cfe/trunk/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp =================================================================== --- cfe/trunk/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp +++ cfe/trunk/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp @@ -0,0 +1,233 @@ +//===- DirectoryWatcher-mac.cpp - Mac-platform directory watching ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Path.h" +#include + +using namespace llvm; +using namespace clang; + +static FSEventStreamRef createFSEventStream( + StringRef Path, + std::function, bool)>, + dispatch_queue_t); +static void stopFSEventStream(FSEventStreamRef); + +namespace { + +class DirectoryWatcherMac : public clang::DirectoryWatcher { +public: + DirectoryWatcherMac( + FSEventStreamRef EventStream, + std::function, bool)> + Receiver, + llvm::StringRef WatchedDirPath) + : EventStream(EventStream), Receiver(Receiver), + WatchedDirPath(WatchedDirPath) {} + + ~DirectoryWatcherMac() override { + stopFSEventStream(EventStream); + EventStream = nullptr; + // Now it's safe to use Receiver as the only other concurrent use would have + // been in EventStream processing. + Receiver(DirectoryWatcher::Event( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""), + false); + } + +private: + FSEventStreamRef EventStream; + std::function, bool)> Receiver; + const std::string WatchedDirPath; +}; + +struct EventStreamContextData { + std::string WatchedPath; + std::function, bool)> Receiver; + + EventStreamContextData( + std::string &&WatchedPath, + std::function, bool)> + Receiver) + : WatchedPath(std::move(WatchedPath)), Receiver(Receiver) {} + + // Needed for FSEvents + static void dispose(const void *ctx) { + delete static_cast(ctx); + } +}; +} // namespace + +constexpr const FSEventStreamEventFlags StreamInvalidatingFlags = + kFSEventStreamEventFlagUserDropped | kFSEventStreamEventFlagKernelDropped | + kFSEventStreamEventFlagMustScanSubDirs; + +constexpr const FSEventStreamEventFlags ModifyingFileEvents = + kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemRenamed | + kFSEventStreamEventFlagItemModified; + +static void eventStreamCallback(ConstFSEventStreamRef Stream, + void *ClientCallBackInfo, size_t NumEvents, + void *EventPaths, + const FSEventStreamEventFlags EventFlags[], + const FSEventStreamEventId EventIds[]) { + auto *ctx = static_cast(ClientCallBackInfo); + + std::vector Events; + for (size_t i = 0; i < NumEvents; ++i) { + StringRef Path = ((const char **)EventPaths)[i]; + const FSEventStreamEventFlags Flags = EventFlags[i]; + + if (Flags & StreamInvalidatingFlags) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } else if (!(Flags & kFSEventStreamEventFlagItemIsFile)) { + // Subdirectories aren't supported - if some directory got removed it + // must've been the watched directory itself. + if ((Flags & kFSEventStreamEventFlagItemRemoved) && + Path == ctx->WatchedPath) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}); + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } + // No support for subdirectories - just ignore everything. + continue; + } else if (Flags & kFSEventStreamEventFlagItemRemoved) { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + continue; + } else if (Flags & ModifyingFileEvents) { + if (!getFileStatus(Path).hasValue()) { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + } else { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Modified, + llvm::sys::path::filename(Path)); + } + continue; + } + + // default + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + llvm_unreachable("Unknown FSEvent type."); + } + + if (!Events.empty()) { + ctx->Receiver(Events, /*IsInitial=*/false); + } +} + +FSEventStreamRef createFSEventStream( + StringRef Path, + std::function, bool)> Receiver, + dispatch_queue_t Queue) { + if (Path.empty()) + return nullptr; + + CFMutableArrayRef PathsToWatch = [&]() { + CFMutableArrayRef PathsToWatch = + CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); + CFStringRef CfPathStr = + CFStringCreateWithBytes(nullptr, (const UInt8 *)Path.data(), + Path.size(), kCFStringEncodingUTF8, false); + CFArrayAppendValue(PathsToWatch, CfPathStr); + CFRelease(CfPathStr); + return PathsToWatch; + }(); + + FSEventStreamContext Context = [&]() { + std::string RealPath; + { + SmallString<128> Storage; + StringRef P = llvm::Twine(Path).toNullTerminatedStringRef(Storage); + char Buffer[PATH_MAX]; + if (::realpath(P.begin(), Buffer) != nullptr) + RealPath = Buffer; + else + RealPath = Path; + } + + FSEventStreamContext Context; + Context.version = 0; + Context.info = new EventStreamContextData(std::move(RealPath), Receiver); + Context.retain = nullptr; + Context.release = EventStreamContextData::dispose; + Context.copyDescription = nullptr; + return Context; + }(); + + FSEventStreamRef Result = FSEventStreamCreate( + nullptr, eventStreamCallback, &Context, PathsToWatch, + kFSEventStreamEventIdSinceNow, /* latency in seconds */ 0.0, + kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagNoDefer); + CFRelease(PathsToWatch); + + return Result; +} + +void stopFSEventStream(FSEventStreamRef EventStream) { + if (!EventStream) + return; + FSEventStreamStop(EventStream); + FSEventStreamInvalidate(EventStream); + FSEventStreamRelease(EventStream); +} + +std::unique_ptr clang::DirectoryWatcher::create( + StringRef Path, + std::function, bool)> Receiver, + bool WaitForInitialSync) { + dispatch_queue_t Queue = + dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); + + if (Path.empty()) + return nullptr; + + auto EventStream = createFSEventStream(Path, Receiver, Queue); + if (!EventStream) { + return nullptr; + } + + std::unique_ptr Result = + llvm::make_unique(EventStream, Receiver, Path); + + // We need to copy the data so the lifetime is ok after a const copy is made + // for the block. + const std::string CopiedPath = Path; + + auto InitWork = ^{ + // We need to start watching the directory before we start scanning in order + // to not miss any event. By dispatching this on the same serial Queue as + // the FSEvents will be handled we manage to start watching BEFORE the + // inital scan and handling events ONLY AFTER the scan finishes. + FSEventStreamSetDispatchQueue(EventStream, Queue); + FSEventStreamStart(EventStream); + // We need to decrement the ref count for Queue as initialize() will return + // and FSEvents has incremented it. Since we have to wait for FSEvents to + // take ownership it's the easiest to do it here rather than main thread. + dispatch_release(Queue); + Receiver(getAsFileEvents(scanDirectory(CopiedPath)), /*IsInitial=*/true); + }; + + if (WaitForInitialSync) { + dispatch_sync(Queue, InitWork); + } else { + dispatch_async(Queue, InitWork); + } + + return Result; +} Index: cfe/trunk/unittests/CMakeLists.txt =================================================================== --- cfe/trunk/unittests/CMakeLists.txt +++ cfe/trunk/unittests/CMakeLists.txt @@ -30,6 +30,7 @@ if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) add_subdirectory(libclang) endif() +add_subdirectory(DirectoryWatcher) add_subdirectory(Rename) add_subdirectory(Index) add_subdirectory(Serialization) Index: cfe/trunk/unittests/DirectoryWatcher/CMakeLists.txt =================================================================== --- cfe/trunk/unittests/DirectoryWatcher/CMakeLists.txt +++ cfe/trunk/unittests/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,13 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_unittest(DirectoryWatcherTests + DirectoryWatcherTest.cpp + ) + +target_link_libraries(DirectoryWatcherTests + PRIVATE + clangDirectoryWatcher + clangBasic + ) \ No newline at end of file Index: cfe/trunk/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp =================================================================== --- cfe/trunk/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp +++ cfe/trunk/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp @@ -0,0 +1,426 @@ +//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include "gtest/gtest.h" +#include +#include +#include +#include + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::sys::fs; +using namespace clang; + +namespace clang { +static bool operator==(const DirectoryWatcher::Event &lhs, + const DirectoryWatcher::Event &rhs) { + return lhs.Filename == rhs.Filename && + static_cast(lhs.Kind) == static_cast(rhs.Kind); +} +} // namespace clang + +namespace { + +struct DirectoryWatcherTestFixture { + std::string TestRootDir; + std::string TestWatchedDir; + + DirectoryWatcherTestFixture() { + SmallString<128> pathBuf; + std::error_code UniqDirRes = createUniqueDirectory("dirwatcher", pathBuf); + assert(!UniqDirRes); + TestRootDir = pathBuf.str(); + path::append(pathBuf, "watch"); + TestWatchedDir = pathBuf.str(); + std::error_code CreateDirRes = create_directory(TestWatchedDir, false); + assert(!CreateDirRes); + } + + ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); } + + SmallString<128> getPathInWatched(const std::string &testFile) { + SmallString<128> pathBuf; + pathBuf = TestWatchedDir; + path::append(pathBuf, testFile); + return pathBuf; + } + + void addFile(const std::string &testFile) { + Expected ft = openNativeFileForWrite(getPathInWatched(testFile), + CD_CreateNew, OF_None); + if (ft) { + closeFile(*ft); + } else { + llvm::errs() << llvm::toString(ft.takeError()) << "\n"; + llvm::errs() << getPathInWatched(testFile) << "\n"; + llvm_unreachable("Couldn't create test file."); + } + } + + void deleteFile(const std::string &testFile) { + std::error_code EC = + remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false); + ASSERT_FALSE(EC); + } +}; + +std::string eventKindToString(const DirectoryWatcher::Event::EventKind K) { + switch (K) { + case DirectoryWatcher::Event::EventKind::Removed: + return "Removed"; + case DirectoryWatcher::Event::EventKind::Modified: + return "Modified"; + case DirectoryWatcher::Event::EventKind::WatchedDirRemoved: + return "WatchedDirRemoved"; + case DirectoryWatcher::Event::EventKind::WatcherGotInvalidated: + return "WatcherGotInvalidated"; + } + llvm_unreachable("unknown event kind"); +} + +struct VerifyingConsumer { + std::vector ExpectedInitial; + std::vector ExpectedNonInitial; + std::vector OptionalNonInitial; + std::vector UnexpectedInitial; + std::vector UnexpectedNonInitial; + std::mutex Mtx; + std::condition_variable ResultIsReady; + + VerifyingConsumer( + const std::vector &ExpectedInitial, + const std::vector &ExpectedNonInitial, + const std::vector &OptionalNonInitial = {}) + : ExpectedInitial(ExpectedInitial), + ExpectedNonInitial(ExpectedNonInitial), + OptionalNonInitial(OptionalNonInitial) {} + + // This method is used by DirectoryWatcher. + void consume(DirectoryWatcher::Event E, bool IsInitial) { + if (IsInitial) + consumeInitial(E); + else + consumeNonInitial(E); + } + + void consumeInitial(DirectoryWatcher::Event E) { + std::unique_lock L(Mtx); + auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E); + if (It == ExpectedInitial.end()) { + UnexpectedInitial.push_back(E); + } else { + ExpectedInitial.erase(It); + } + if (result()) + ResultIsReady.notify_one(); + } + + void consumeNonInitial(DirectoryWatcher::Event E) { + std::unique_lock L(Mtx); + auto It = + std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E); + if (It == ExpectedNonInitial.end()) { + auto OptIt = + std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E); + if (OptIt != OptionalNonInitial.end()) { + OptionalNonInitial.erase(OptIt); + } else { + UnexpectedNonInitial.push_back(E); + } + } else { + ExpectedNonInitial.erase(It); + } + if (result()) + ResultIsReady.notify_one(); + } + + // This method is used by DirectoryWatcher. + void consume(llvm::ArrayRef Es, bool IsInitial) { + for (const auto &E : Es) + consume(E, IsInitial); + } + + // Not locking - caller has to lock Mtx. + llvm::Optional result() const { + if (ExpectedInitial.empty() && ExpectedNonInitial.empty() && + UnexpectedInitial.empty() && UnexpectedNonInitial.empty()) + return true; + if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty()) + return false; + return llvm::None; + } + + // This method is used by tests. + // \returns true on success + bool blockUntilResult() { + std::unique_lock L(Mtx); + while (true) { + if (result()) + return *result(); + + ResultIsReady.wait(L, [this]() { return result().hasValue(); }); + } + return false; // Just to make compiler happy. + } + + void printUnmetExpectations(llvm::raw_ostream &OS) { + if (!ExpectedInitial.empty()) { + OS << "Expected but not seen initial events: \n"; + for (const auto &E : ExpectedInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!ExpectedNonInitial.empty()) { + OS << "Expected but not seen non-initial events: \n"; + for (const auto &E : ExpectedNonInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!UnexpectedInitial.empty()) { + OS << "Unexpected initial events seen: \n"; + for (const auto &E : UnexpectedInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!UnexpectedNonInitial.empty()) { + OS << "Unexpected non-initial events seen: \n"; + for (const auto &E : UnexpectedNonInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + } +}; + +void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) { + std::packaged_task task( + [&TestConsumer]() { return TestConsumer.blockUntilResult(); }); + std::future WaitForExpectedStateResult = task.get_future(); + std::thread worker(std::move(task)); + worker.detach(); + + EXPECT_TRUE(WaitForExpectedStateResult.wait_for(std::chrono::seconds(3)) == + std::future_status::ready) + << "The expected result state wasn't reached before the time-out."; + EXPECT_TRUE(TestConsumer.result().hasValue()); + if (TestConsumer.result().hasValue()) { + EXPECT_TRUE(*TestConsumer.result()); + } + if ((TestConsumer.result().hasValue() && !TestConsumer.result().getValue()) || + !TestConsumer.result().hasValue()) + TestConsumer.printUnmetExpectations(llvm::outs()); +} + +} // namespace + +TEST(DirectoryWatcherTest, InitialScanSync) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}, + {}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, InitialScanAsync) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}, + {}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/false); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, AddFiles) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, ModifyFile) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + // modify the file + { + std::error_code error; + llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error, + CD_OpenExisting); + assert(!error); + bStream << "foo"; + } + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, DeleteFile) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + {{DirectoryWatcher::Event::EventKind::Removed, "a"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + fixture.deleteFile("a"); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, DeleteWatchedDir) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, + {{DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}, + {DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + remove_directories(fixture.TestWatchedDir); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, InvalidatedWatcher) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, {{DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; + + { + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + } // DW is destructed here. + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, ChangeMetadata) { + DirectoryWatcherTestFixture fixture; + fixture.addFile("a"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + // We don't expect any notification for file having access file changed. + {}, + // Given the timing we are ok with receiving the duplicate event. + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + { // Change access and modification time of file a. + Expected HopefullyTheFD = llvm::sys::fs::openNativeFileForWrite( + fixture.getPathInWatched("a"), CD_OpenExisting, OF_None); + if (!HopefullyTheFD) { + llvm::outs() << HopefullyTheFD.takeError(); + } + + const int FD = HopefullyTheFD.get(); + const TimePoint<> NewTimePt = + std::chrono::system_clock::now() - std::chrono::minutes(1); + + std::error_code setTimeRes = + llvm::sys::fs::setLastAccessAndModificationTime(FD, NewTimePt, + NewTimePt); + assert(!setTimeRes); + } + + checkEventualResultWithTimeout(TestConsumer); +}