jkorous updated this revision to Diff 199712. jkorous added a comment. A major clean-up.
changelog ========= **general** - specification, documentation, tests - returning only filenames instead of paths (or empty string when the event is related to the watched dir itself) - removed unsound event deduplication during/right after the initial scan - simplified how is OS-specific implementation selected **linux** - properly terminating threads - added pthreads to fix shared libs build - handle IN_DELETE_SELF, IN_IGNORED - IN_EXCL_UNLINK **macos** - simplified synchronization by removing semaphores - workarounds in FSEvents use I am not entirely happy about my tests - since we're handling notifications from kernel asynchronously it's hard to write a deterministic test. I just use some 100 ms sleeps as a workaround. Happy to use something more robust if anyone has some ideas. Tests seem fine with msan & tsan on linux and tsan on macos but given their scope it doesn't mean that much. CHANGES SINCE LAST ACTION https://reviews.llvm.org/D58418/new/ https://reviews.llvm.org/D58418 Files: clang/cmake/modules/AddClang.cmake clang/include/clang/DirectoryWatcher/DirectoryWatcher.h clang/lib/CMakeLists.txt clang/lib/DirectoryWatcher/CMakeLists.txt clang/lib/DirectoryWatcher/DirectoryScanner.cpp clang/lib/DirectoryWatcher/DirectoryScanner.h clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp clang/unittests/CMakeLists.txt clang/unittests/DirectoryWatcher/CMakeLists.txt clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp
Index: clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp =================================================================== --- /dev/null +++ clang/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp @@ -0,0 +1,359 @@ +//===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include "gtest/gtest.h" +#include <condition_variable> +#include <mutex> +#include <thread> + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::sys::fs; +using namespace clang; + +namespace clang { +static bool operator==(const DirectoryWatcher::Event &lhs, + const DirectoryWatcher::Event &rhs) { + return lhs.Filename == rhs.Filename && + static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind); +} +} // namespace clang + +namespace { + +// Intentionally trivial - expecting just a very small numbers of events. +class TestEventConsumer { + std::vector<DirectoryWatcher::Event> InitialEvents; + std::vector<DirectoryWatcher::Event> NonInitialEvents; + sys::Mutex Mtx; + + // Expecting only very small sets - don't bother with anything smart. + static bool + AreExpectedPresent(const std::vector<DirectoryWatcher::Event> &actual, + const std::vector<DirectoryWatcher::Event> &expected) { + for (const auto &e : expected) { + if (std::find(actual.begin(), actual.end(), e) == actual.end()) + return false; + } + return true; + } + +public: + void push(llvm::ArrayRef<DirectoryWatcher::Event> Events, bool isInitial) { + sys::ScopedLock L(Mtx); + if (isInitial) { + for (const auto &E : Events) + InitialEvents.push_back(E); + } else { + for (const auto &E : Events) + NonInitialEvents.push_back(E); + } + } + + std::vector<DirectoryWatcher::Event> getInitialEvents() { + sys::ScopedLock L(Mtx); + return InitialEvents; + } + std::vector<DirectoryWatcher::Event> getNonInitialEvents() { + sys::ScopedLock L(Mtx); + return NonInitialEvents; + } + + // Fool-proof way how to compare. + bool AreExpectedPresentInInitial( + const std::vector<DirectoryWatcher::Event> &expected) { + sys::ScopedLock L(Mtx); + return AreExpectedPresent(InitialEvents, expected); + } + + // Fool-proof way how to compare. + bool AreExpectedPresentInNonInitial( + const std::vector<DirectoryWatcher::Event> &expected) { + sys::ScopedLock L(Mtx); + return AreExpectedPresent(NonInitialEvents, expected); + } +}; + +struct DirectoryWatcherTestFixture { + std::string TestRootDir; + std::string TestWatchedDir; + + DirectoryWatcherTestFixture() { + SmallString<128> pathBuf; + assert(!createUniqueDirectory("dirwatcher", pathBuf)); + TestRootDir = pathBuf.str(); + path::append(pathBuf, "watch"); + TestWatchedDir = pathBuf.str(); + assert(!create_directory(TestWatchedDir, false)); + } + + ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); } + + SmallString<128> getPathInWatched(const std::string &testFile) { + SmallString<128> pathBuf; + pathBuf = TestWatchedDir; + path::append(pathBuf, testFile); + return pathBuf; + } + + void addFile(const std::string &testFile) { + Expected<file_t> ft = openNativeFileForWrite(getPathInWatched(testFile), + CD_CreateNew, OF_None); + if (ft) { + closeFile(*ft); + } else { + llvm::errs() << llvm::toString(ft.takeError()) << "\n"; + llvm::errs() << getPathInWatched(testFile) << "\n"; + llvm_unreachable("Couldn't create test file."); + } + } + + void deleteFile(const std::string &testFile) { + std::error_code EC = + remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false); + ASSERT_FALSE(EC); + } +}; + +void WaitForAsync() { + // This is just a heuristic - trying to wait for changes to FS to propagate so + // the async event handling picks them up. Can make this test flaky. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // 0.1s +} + +} // namespace + +TEST(DirectoryWatcherTest, initialScanSync) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + assert(error.empty()); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 3u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 0u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInInitial( + {{DirectoryWatcher::Event::EventKind::Added, "a"}, + {DirectoryWatcher::Event::EventKind::Added, "b"}, + {DirectoryWatcher::Event::EventKind::Added, "c"}})); +} + +TEST(DirectoryWatcherTest, initialScanAsync) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + // Don't want duplicate events during initial scan. + WaitForAsync(); + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/false, error); + assert(error.empty()); + + // Wait for async scan to finish. + WaitForAsync(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 3u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 0u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInInitial( + {{DirectoryWatcher::Event::EventKind::Added, "a"}, + {DirectoryWatcher::Event::EventKind::Added, "b"}, + {DirectoryWatcher::Event::EventKind::Added, "c"}})); +} + +TEST(DirectoryWatcherTest, addFiles) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + assert(error.empty()); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 0u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 0u); + + // Wait for async events to be processed. + WaitForAsync(); + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + // Wait for async events to be processed. + WaitForAsync(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 0u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 3u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInNonInitial( + {{DirectoryWatcher::Event::EventKind::Added, "a"}, + {DirectoryWatcher::Event::EventKind::Added, "b"}, + {DirectoryWatcher::Event::EventKind::Added, "c"}})); +} + +TEST(DirectoryWatcherTest, modifyFile) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + fixture.addFile("b"); + + WaitForAsync(); + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + assert(error.empty()); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 1u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 0u); + + // modify the file + { + std::error_code error; + llvm::raw_fd_ostream bStream(fixture.getPathInWatched("b"), error, + CD_OpenExisting); + assert(!error); + bStream << "foo"; + } + + WaitForAsync(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 1u); + EXPECT_GT(eventConsumer.getNonInitialEvents().size(), 0u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInNonInitial( + {{DirectoryWatcher::Event::EventKind::Modified, "b"}})); +} + +TEST(DirectoryWatcherTest, deleteFile) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + WaitForAsync(); + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + assert(error.empty()); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 3u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 0u); + + fixture.deleteFile("b"); + + WaitForAsync(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 3u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 1u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInNonInitial( + {{DirectoryWatcher::Event::EventKind::Removed, "b"}})); +} + +TEST(DirectoryWatcherTest, deleteWatchedDir) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + std::string error; + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + assert(error.empty()); + + remove_directories(fixture.TestWatchedDir); + + WaitForAsync(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 0u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 2u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInNonInitial( + {{DirectoryWatcher::Event::EventKind::DirectoryDeleted, ""}, + {DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}})); +} + +TEST(DirectoryWatcherTest, InvalidatedWatcher) { + DirectoryWatcherTestFixture fixture; + TestEventConsumer eventConsumer; + + std::string error; + + auto receiver = + [&eventConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events, + bool isInitial) { + eventConsumer.push(Events, isInitial); + }; + + auto DW = createDirectoryWatcher(fixture.TestWatchedDir, receiver, + /*waitInitialSync=*/true, error); + WaitForAsync(); + DW.reset(); + + EXPECT_EQ(eventConsumer.getInitialEvents().size(), 0u); + EXPECT_EQ(eventConsumer.getNonInitialEvents().size(), 1u); + + EXPECT_TRUE(eventConsumer.AreExpectedPresentInNonInitial( + {{DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}})); +} \ No newline at end of file Index: clang/unittests/DirectoryWatcher/CMakeLists.txt =================================================================== --- /dev/null +++ clang/unittests/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_unittest(DirectoryWatcherTests + DirectoryWatcherTest.cpp + ) + +target_link_libraries(DirectoryWatcherTests + PRIVATE + clangDirectoryWatcher + clangBasic + ) + +if(APPLE) + check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES_H) + if(HAVE_CORESERVICES_H) + set(DWT_LINK_FLAGS "${DWT_LINK_FLAGS} -framework CoreServices") + set_property(TARGET DirectoryWatcherTests APPEND_STRING PROPERTY + LINK_FLAGS ${DWT_LINK_FLAGS}) + endif() +endif() Index: clang/unittests/CMakeLists.txt =================================================================== --- clang/unittests/CMakeLists.txt +++ clang/unittests/CMakeLists.txt @@ -30,5 +30,6 @@ if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) add_subdirectory(libclang) endif() +add_subdirectory(DirectoryWatcher) add_subdirectory(Rename) add_subdirectory(Index) Index: clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp @@ -0,0 +1,228 @@ +//===- DirectoryWatcher-mac.cpp - Mac-platform directory watching ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Path.h" +#include <CoreServices/CoreServices.h> + +using namespace llvm; +using namespace clang; + +static FSEventStreamRef createFSEventStream(StringRef Path, + DirectoryWatcher::EventReceiver, + dispatch_queue_t); +static void stopFSEventStream(FSEventStreamRef); + +namespace { + +class DirectoryWatcherMac : public clang::DirectoryWatcher { +public: + DirectoryWatcherMac(FSEventStreamRef EventStream, + DirectoryWatcher::EventReceiver Receiver, + llvm::StringRef WatchedDirPath) + : EventStream(EventStream), Receiver(Receiver), + WatchedDirPath(WatchedDirPath) {} + + ~DirectoryWatcherMac() override { + stopFSEventStream(EventStream); + EventStream = nullptr; + // Now it's safe to use Receiver as the only other concurrent use would have + // been in EventStream processing. + Receiver(DirectoryWatcher::Event( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""), + false); + } + +private: + FSEventStreamRef EventStream; + DirectoryWatcher::EventReceiver Receiver; + const std::string WatchedDirPath; +}; + +struct EventStreamContextData { + std::string WatchedPath; + DirectoryWatcher::EventReceiver Receiver; + + EventStreamContextData(std::string &&WatchedPath, + DirectoryWatcher::EventReceiver Receiver) + : WatchedPath(std::move(WatchedPath)), Receiver(Receiver) {} + + // Needed for FSEvents + static void dispose(const void *ctx) { + delete static_cast<const EventStreamContextData *>(ctx); + } +}; +} // namespace + +constexpr const FSEventStreamEventFlags StreamInvalidatingFlags = + kFSEventStreamEventFlagUserDropped | kFSEventStreamEventFlagKernelDropped | + kFSEventStreamEventFlagMustScanSubDirs; + +static void eventStreamCallback(ConstFSEventStreamRef Stream, + void *ClientCallBackInfo, size_t NumEvents, + void *EventPaths, + const FSEventStreamEventFlags EventFlags[], + const FSEventStreamEventId EventIds[]) { + auto *ctx = static_cast<EventStreamContextData *>(ClientCallBackInfo); + + std::vector<DirectoryWatcher::Event> Events; + for (size_t i = 0; i < NumEvents; ++i) { + StringRef Path = ((const char **)EventPaths)[i]; + const FSEventStreamEventFlags Flags = EventFlags[i]; + + if (Flags & StreamInvalidatingFlags) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } else if (!(Flags & kFSEventStreamEventFlagItemIsFile)) { + // Subdirectories aren't supported - if some directory got removed it + // must've been the watched directory itself. + if ((Flags & kFSEventStreamEventFlagItemRemoved) && + Path == ctx->WatchedPath) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::DirectoryDeleted, ""}); + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } + } else if (Flags & kFSEventStreamEventFlagItemModified) { + // WORKAROUND: Event can have both kFSEventStreamEventFlagItemModified and + // kFSEventStreamEventFlagItemCreated flags set for a file that was + // modified - we must check for flag Modified first. + // rdar://problem/50825999 + Events.emplace_back(DirectoryWatcher::Event::EventKind::Modified, + llvm::sys::path::filename(Path)); + } else if (Flags & (kFSEventStreamEventFlagItemCreated | + kFSEventStreamEventFlagItemRenamed)) { + // NOTE: With low latency sometimes for a file that is moved inside the + // directory, or for a file that is removed from the directory, the flags + // have both 'renamed' and 'removed'. We use getting the file status as a + // way to distinguish between the two. + if (!getFileStatus(Path)) + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + else + Events.emplace_back(DirectoryWatcher::Event::EventKind::Added, + llvm::sys::path::filename(Path)); + } else if (Flags & kFSEventStreamEventFlagItemRemoved) { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + } else { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + llvm_unreachable("Unknown FSEvent type."); + } + } + + if (!Events.empty()) { + ctx->Receiver(Events, /*isInitial=*/false); + } +} + +FSEventStreamRef +createFSEventStream(StringRef Path, + clang::DirectoryWatcher::EventReceiver Receiver, + dispatch_queue_t Queue) { + if (Path.empty()) + return nullptr; + + CFMutableArrayRef PathsToWatch = [&]() { + CFMutableArrayRef PathsToWatch = + CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); + CFStringRef CfPathStr = + CFStringCreateWithBytes(nullptr, (const UInt8 *)Path.data(), + Path.size(), kCFStringEncodingUTF8, false); + CFArrayAppendValue(PathsToWatch, CfPathStr); + CFRelease(CfPathStr); + return PathsToWatch; + }(); + + FSEventStreamContext Context = [&]() { + std::string RealPath; + { + SmallString<128> Storage; + StringRef P = llvm::Twine(Path).toNullTerminatedStringRef(Storage); + char Buffer[PATH_MAX]; + if (::realpath(P.begin(), Buffer) != nullptr) + RealPath = Buffer; + else + RealPath = Path; + } + + FSEventStreamContext Context; + Context.version = 0; + Context.info = new EventStreamContextData(std::move(RealPath), Receiver); + Context.retain = nullptr; + Context.release = EventStreamContextData::dispose; + Context.copyDescription = nullptr; + return Context; + }(); + + FSEventStreamRef Result = FSEventStreamCreate( + nullptr, eventStreamCallback, &Context, PathsToWatch, + kFSEventStreamEventIdSinceNow, /* latency in seconds */ 0.0, + kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagNoDefer); + CFRelease(PathsToWatch); + + return Result; +} + +void stopFSEventStream(FSEventStreamRef EventStream) { + if (!EventStream) + return; + FSEventStreamStop(EventStream); + FSEventStreamInvalidate(EventStream); + FSEventStreamRelease(EventStream); +} + +std::unique_ptr<DirectoryWatcher> +clang::createDirectoryWatcher(StringRef Path, + DirectoryWatcher::EventReceiver Receiver, + bool WaitInitialSync, std::string &Error) { + dispatch_queue_t Queue = + dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); + + auto EventStream = createFSEventStream(Path, Receiver, Queue); + if (!EventStream) { + raw_string_ostream(Error) + << "failed to setup FSEvents stream for path: " << Path; + return nullptr; + } + + std::unique_ptr<DirectoryWatcher> Result( + new DirectoryWatcherMac(EventStream, Receiver, Path)); + + // We need to copy the data so the lifetime is ok after a const copy is made + // for the block. + const std::string CopiedPath = Path; + + auto InitWork = ^{ + // We need to start watching the directory before we start scanning in order + // to not miss any event. By dispatching this on the same serial Queue as + // the FSEvents will be handled we manage to start watching BEFORE the + // inital scan and handling events ONLY AFTER the scan finishes. + FSEventStreamSetDispatchQueue(EventStream, Queue); + FSEventStreamStart(EventStream); + // We need to decrement the ref count for Queue as initialize() will return + // and FSEvents has incremented it. Since we have to wait for FSEvents to + // take ownership it's the easiest to do it here rather than main thread. + dispatch_release(Queue); + Receiver(getAsFileEvents(scanDirectory(CopiedPath)), /*isInitial=*/true); + }; + + if (WaitInitialSync) { + dispatch_sync(Queue, InitWork); + } else { + dispatch_async(Queue, InitWork); + } + + return Result; +} Index: clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp @@ -0,0 +1,273 @@ +//===- DirectoryWatcher-linux.cpp - Linux-platform directory watching -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/Support/Errno.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include <atomic> +#include <queue> +#include <string> +#include <thread> +#include <vector> + +#include <poll.h> +#include <sys/inotify.h> +#include <unistd.h> + +namespace { + +using namespace llvm; +using namespace clang; + +// Simple mutex-protected queue of Events. +class EventQueue { + sys::Mutex Mtx; + std::queue<DirectoryWatcher::Event> Events; + +public: + void push_back(const DirectoryWatcher::Event::EventKind K, + StringRef Filename) { + sys::ScopedLock L(Mtx); + Events.emplace(K, Filename); + } + + llvm::Optional<DirectoryWatcher::Event> pop_front() { + sys::ScopedLock L(Mtx); + if (Events.empty()) { + return llvm::None; + } else { + DirectoryWatcher::Event Front = Events.front(); + Events.pop(); + return Front; + } + } +}; + +class DirectoryWatcherLinux : public clang::DirectoryWatcher { +public: + DirectoryWatcherLinux(llvm::StringRef WatchedDirPath, + DirectoryWatcher::EventReceiver Receiver, + bool WaitInitialSync, int InotifyFD, int InotifyWD); + + ~DirectoryWatcherLinux() override { + Stop = true; + InotifyPollingThread.join(); + ReceivingThread.join(); + // Now it's safe to use Receiver as the only other concurrent use would have + // been in ReceivingThread. + Receiver(DirectoryWatcher::Event( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""), + false); + inotify_rm_watch(InotifyFD, InotifyWD); + llvm::sys::RetryAfterSignal(-1, close, InotifyFD); + } + +private: + void HandleFailureInPollingThread() { + Queue.push_back(DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + ""); + Failure = true; + Stop = true; + } + + // inotify file descriptor + int InotifyFD = -1; + // inotify watch descriptor + int InotifyWD = -1; + EventQueue Queue; + // flag used for stopping all async acions + std::atomic<bool> Stop; + // If failure happens in the polling thread we need to notify client by + // sending him WatcherGotInvalidated event in the ReceivingThread. + std::atomic<bool> Failure; + std::atomic<bool> FinishedInitScan; + // Make sure lifetime of Receiver fully contains lifetime of ReceivingThread. + DirectoryWatcher::EventReceiver Receiver; + + std::thread InotifyPollingThread; + // Does the initial scan of the directory and then keeps processing events + // received from inotify. + std::thread ReceivingThread; +}; + +DirectoryWatcherLinux::DirectoryWatcherLinux( + StringRef WatchedDirPath, DirectoryWatcher::EventReceiver Receiver, + bool WaitInitialSync, int InotifyFD, int InotifyWD) + : InotifyFD(InotifyFD), InotifyWD(InotifyWD), Stop(false), Failure(false), + FinishedInitScan(false), Receiver(Receiver), + InotifyPollingThread([this, InotifyFD]() { + // We want to be able to read ~30 events at once even in the worst case + // (obscenely long filenames). + constexpr size_t EventBufferLength = + 30 * (sizeof(struct inotify_event) + NAME_MAX + 1); + // http://man7.org/linux/man-pages/man7/inotify.7.html + /* Some systems cannot read integer variables if they are not + properly aligned. On other systems, incorrect alignment may + decrease performance. Hence, the buffer used for reading from + the inotify file descriptor should have the same alignment as + struct inotify_event. */ + char Buf[EventBufferLength] + __attribute__((aligned(__alignof__(struct inotify_event)))); + + struct pollfd PollReq; + PollReq.fd = InotifyFD; + PollReq.events = POLLIN; + + const int TimeoutMs = 1; + + while (true) { + if (Stop) + return; + + // Keep checking if we received any new event but don't block forever + // and check the terminating variable. + while (true) { + if (Stop) + return; + + const int PollResult = poll(&PollReq, 1, TimeoutMs); + // There are inotify events waiting to be read! + if (PollResult == 1 && PollReq.revents == POLLIN) + break; + + // We just hit the timeout - get back to waiting. + if (PollResult == 0) + continue; + + // We just got interrupted by a signal - get back to waiting for + // inotify events. + if (PollResult == -1 && errno == EINTR) + continue; + + // Something is badly wrong - e. g. EFAULT or POLLERR. + HandleFailureInPollingThread(); + return; + } + + ssize_t NumRead = llvm::sys::RetryAfterSignal( + -1, read, InotifyFD, reinterpret_cast<char *>(Buf), + EventBufferLength); + + for (char *p = Buf; p < Buf + NumRead;) { + if (p + sizeof(struct inotify_event) > Buf + NumRead) { + HandleFailureInPollingThread(); + llvm_unreachable("an incomplete inotify_event was read"); + return; + } + + struct inotify_event *Event = + reinterpret_cast<struct inotify_event *>(p); + p += sizeof(struct inotify_event) + Event->len; + + if (Event->mask & + (IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_DELETE) && + Event->len <= 0) { + HandleFailureInPollingThread(); + llvm_unreachable("expected a filename from inotify"); + return; + } + + if (Event->mask & IN_MODIFY) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Modified, + Event->name); + } else if (Event->mask & (IN_CREATE | IN_MOVED_TO)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Added, + Event->name); + } else if (Event->mask & (IN_DELETE | IN_MOVED_FROM)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Removed, + Event->name); + } else if (Event->mask & (IN_DELETE_SELF | IN_MOVE_SELF)) { + Queue.push_back( + DirectoryWatcher::Event::EventKind::DirectoryDeleted, ""); + Queue.push_back( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + ""); + Stop = true; + return; + } else if (Event->mask & IN_IGNORED) { + Queue.push_back( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + ""); + Stop = true; + return; + } else { + HandleFailureInPollingThread(); + llvm_unreachable("Unknown event type."); + return; + } + } + } + }), + ReceivingThread([this, WatchedDirPath]() { + // Initial scan of watched directory first ... + this->Receiver(getAsFileEvents(scanDirectory(WatchedDirPath)), + /*isInitial=*/true); + + FinishedInitScan = true; + + // ... inotify-originated events processing ever after. + while (true) { + bool GotEvent = false; + // By processing all the events in the queue we make sure that we + // don't miss any important closing event (DirectoryDeleted etc). + while (llvm::Optional<DirectoryWatcher::Event> MaybeEvent = + this->Queue.pop_front()) { + this->Receiver(MaybeEvent.getValue(), false); + GotEvent = true; + }; + if (!GotEvent) { + if (Stop) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + + if (Failure) { + this->Receiver( + {DirectoryWatcher::Event( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + "")}, + false); + } + }) { + while (WaitInitialSync && !Stop && !FinishedInitScan) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +} // namespace + +std::unique_ptr<clang::DirectoryWatcher> +clang::createDirectoryWatcher(StringRef Path, + DirectoryWatcher::EventReceiver Receiver, + bool WaitInitialSync, std::string &ErrorMsg) { + auto error = [&](StringRef msg) { + ErrorMsg = msg; + ErrorMsg += ": "; + ErrorMsg += llvm::sys::StrError(); + return nullptr; + }; + + const int InotifyFD = inotify_init(); + + if (InotifyFD == -1) + return error("inotify_init failed"); + + const int InotifyWD = inotify_add_watch( + InotifyFD, Path.str().c_str(), + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_EXCL_UNLINK | IN_MODIFY | + IN_MOVED_FROM | IN_MOVE_SELF | IN_MOVED_TO | IN_ONLYDIR | IN_IGNORED); + if (InotifyWD == -1) + return error("inotify_add_watch failed"); + + return std::unique_ptr<clang::DirectoryWatcher>(new DirectoryWatcherLinux( + Path, Receiver, WaitInitialSync, InotifyFD, InotifyWD)); +} \ No newline at end of file Index: clang/lib/DirectoryWatcher/DirectoryScanner.h =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/DirectoryScanner.h @@ -0,0 +1,29 @@ +//===- DirectoryScanner.h - Utility functions for DirectoryWatcher --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include <string> +#include <vector> + +namespace clang { + +/// Gets names (filenames) of items in directory at \p Path. +/// \returns empty vector if \p Path is not a directory, doesn't exist or can't +/// be read from. +std::vector<std::string> scanDirectory(llvm::StringRef Path); + +/// Create event with EventKind::Added for every element in \p Scan. +std::vector<DirectoryWatcher::Event> +getAsFileEvents(const std::vector<std::string> &Scan); + +/// Gets status of file (or directory) at \p Path. +/// \returns llvm::None if \p Path doesn't exist or can't get the status. +llvm::Optional<llvm::sys::fs::file_status> getFileStatus(llvm::StringRef Path); + +} // namespace clang \ No newline at end of file Index: clang/lib/DirectoryWatcher/DirectoryScanner.cpp =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/DirectoryScanner.cpp @@ -0,0 +1,54 @@ +//===- DirectoryScanner.cpp - Utility functions for DirectoryWatcher ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" + +#include "llvm/Support/Path.h" + +namespace clang { + +using namespace llvm; + +Optional<sys::fs::file_status> getFileStatus(StringRef Path) { + sys::fs::file_status Status; + std::error_code EC = status(Path, Status); + if (EC) + return None; + return Status; +} + +std::vector<std::string> scanDirectory(StringRef Path) { + using namespace llvm::sys; + std::vector<std::string> Result; + + std::error_code EC; + for (auto It = fs::directory_iterator(Path, EC), + End = fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + auto status = getFileStatus(It->path()); + if (!status.hasValue()) + continue; + Result.emplace_back(sys::path::filename(It->path())); + } + + return Result; +} + +std::vector<DirectoryWatcher::Event> +getAsFileEvents(const std::vector<std::string> &Scan) { + std::vector<DirectoryWatcher::Event> Events; + Events.reserve(Scan.size()); + + for (const auto &File : Scan) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::Added, File}); + } + return Events; +} + +} // namespace clang \ No newline at end of file Index: clang/lib/DirectoryWatcher/CMakeLists.txt =================================================================== --- /dev/null +++ clang/lib/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,30 @@ +include(CheckIncludeFiles) + +set(LLVM_LINK_COMPONENTS support) + +set(DIRECTORY_WATCHER_SOURCES DirectoryScanner.cpp) +set(DIRECTORY_WATCHER_LINK_LIBS "") +set(DIRECTORY_WATCHER_LINK_FLAGS "") + +if(APPLE) + check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES) + if(HAVE_CORESERVICES) + list(APPEND DIRECTORY_WATCHER_SOURCES mac/DirectoryWatcher-mac.cpp) + set(DIRECTORY_WATCHER_LINK_FLAGS "${DIRECTORY_WATCHER_LINK_FLAGS} -framework CoreServices") + endif() +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + check_include_files("sys/inotify.h" HAVE_INOTIFY) + if(HAVE_INOTIFY) + list(APPEND DIRECTORY_WATCHER_SOURCES linux/DirectoryWatcher-linux.cpp) + find_package(Threads REQUIRED) + list(APPEND DIRECTORY_WATCHER_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) + endif() +endif() + +add_clang_library(clangDirectoryWatcher + ${DIRECTORY_WATCHER_SOURCES} + ADDITIONAL_LIBS ${DIRECTORY_WATCHER_LINK_LIBS} + ) + +set_property(TARGET clangDirectoryWatcher APPEND_STRING PROPERTY + LINK_FLAGS ${DIRECTORY_WATCHER_LINK_FLAGS}) \ No newline at end of file Index: clang/lib/CMakeLists.txt =================================================================== --- clang/lib/CMakeLists.txt +++ clang/lib/CMakeLists.txt @@ -18,6 +18,7 @@ add_subdirectory(Frontend) add_subdirectory(FrontendTool) add_subdirectory(Tooling) +add_subdirectory(DirectoryWatcher) add_subdirectory(Index) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(StaticAnalyzer) Index: clang/include/clang/DirectoryWatcher/DirectoryWatcher.h =================================================================== --- /dev/null +++ clang/include/clang/DirectoryWatcher/DirectoryWatcher.h @@ -0,0 +1,127 @@ +//===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H +#define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include <functional> +#include <memory> +#include <string> + +namespace clang { +/// Provides notifications for file changes in a directory. +/// +/// Invokes client-provided function on every filesystem event in the watched +/// directory. Initially the the watched directory is scanned and for every file +/// found, an event is synthesized as if the file was added. +/// +/// This is not a general purpose directory monitoring tool - list of +/// limitations follows. +/// +/// Only flat directories with no subdirectories are supported. In case +/// subdirectories are present the behavior is undefined - events *might* be +/// passed to Receiver on macOS (due to FSEvents being used) while they +/// *probably* won't be passed on Linux (due to inotify being used). +/// +/// Known potential inconsistencies +/// - For files that are deleted befor the initial scan processed them, clients +/// might receive Removed notification without any prior Added notification. +/// - Multiple notifications might be produced when a file is added to the +/// watched directory during the initial scan. We are choosing the lesser evil +/// here as the only known alternative strategy would be to invalidate the +/// watcher instance and force user to create a new one whenever filesystem +/// event occurs during the initial scan but that would introduce continuous +/// restarting failure mode (watched directory is not always "owned" by the same +/// process that is consuming it). Since existing clients can handle duplicate +/// events well, we decided for simplicity. +/// +/// Notifications are provided only for changes done through local user-space +/// filesystem interface. Specifically, it's undefined if notification would be +/// provided in case of a: +/// - a file mmap-ed and changed +/// - a file changed via remote (NFS) or virtual (/proc) FS access to monitored +/// directory +/// - another filesystem mounted to the watched directory +/// +/// No support for LLVM VFS. +/// +/// Directories containing "too many" files and/or receiving events "too +/// frequently" are not supported - if the initial scan can't be finished before +/// the watcher instance gets invalidated (see WatcherGotInvalidated) there's no +/// good error handling strategy - the only option for client is to destroy the +/// watcher, restart watching with new instance and hope it won't repeat. +class DirectoryWatcher { +public: + struct Event { + enum class EventKind { + /// A file was added to the directory. + /// If a file gets moved into the directory and replaces an existing file + /// with the same name an 'Added' event will be generated. + /// If a file gets replaced multiple times within a short time period, it + /// may result in only one 'Added' event due to coalescing by the file + /// system notification mechanism. + Added, + /// A file was removed. + Removed, + /// A file was modified. + Modified, + /// The watched directory got deleted. + DirectoryDeleted, + /// The DirectoryWatcher that originated this event is no longer valid and + /// it's behavior is undefined. + /// The prime case is kernel signalling to OS-specific implementation of + /// DirectoryWatcher some resource limit being hit. + /// *Usually* kernel starts dropping or squashing events together after + /// that and so would DirectoryWatcher. This means that *some* events + /// might still be passed to Receiver but this behavior is undefined. + /// The only proper response to this kind of event is to destruct the + /// originating DirectoryWatcher instance and create a new one. + /// Another case is after the watched directory itself is deleted. + /// WatcherGotInvalidated will be received at least once during + /// DirectoryWatcher instance lifetime - when handling errors this is done + /// on best effort basis, when an instance is being destoryed then this is + /// guaranteed. + WatcherGotInvalidated + }; + + EventKind Kind; + /// Filename - a relative path to the watched directory or empty string in + /// case event is related to the directory itself. + std::string Filename; + + Event(EventKind Kind, llvm::StringRef Filename) + : Kind(Kind), Filename(Filename) {} + }; + + typedef std::function<void(llvm::ArrayRef<Event> Events, bool isInitial)> + EventReceiver; + + virtual ~DirectoryWatcher() = default; + DirectoryWatcher(const DirectoryWatcher &) = delete; + DirectoryWatcher &operator=(const DirectoryWatcher &) = delete; + DirectoryWatcher(DirectoryWatcher &&) = delete; + DirectoryWatcher &operator=(DirectoryWatcher &&) = delete; + +protected: + DirectoryWatcher() = default; +}; + +/// Returns nullptr if \param Path doesn't exist. +/// Returns nullptr if \param Path isn't a directory. +/// Returns nullptr if OS kernel API told us we can't start watching. In such +/// case it's unclear whether just retrying has any chance to succeeed. +std::unique_ptr<DirectoryWatcher> +createDirectoryWatcher(llvm::StringRef Path, + DirectoryWatcher::EventReceiver Receiver, + bool WaitInitialSync, std::string &Error); + +} // namespace clang + +#endif // LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H Index: clang/cmake/modules/AddClang.cmake =================================================================== --- clang/cmake/modules/AddClang.cmake +++ clang/cmake/modules/AddClang.cmake @@ -46,7 +46,7 @@ cmake_parse_arguments(ARG "SHARED" "" - "ADDITIONAL_HEADERS" + "ADDITIONAL_HEADERS;ADDITIONAL_LIBS" ${ARGN}) set(srcs) if(MSVC_IDE OR XCODE) @@ -86,7 +86,7 @@ llvm_add_library(${name} ${ARG_ENABLE_SHARED} ${ARG_UNPARSED_ARGUMENTS} ${srcs}) if(TARGET ${name}) - target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS}) + target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS} PRIVATE ${ARG_ADDITIONAL_LIBS}) if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY OR ${name} STREQUAL "libclang")
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
