From 49fb0b43d98b2369930e13bbbace7a96a9378535 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 25 Nov 2021 00:11:25 +0100 Subject: [PATCH 1/8] move to libwatcher --- src/BetterFileWatching.jl | 94 ++++++------------ src/libwatcher.jl | 201 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 66 deletions(-) create mode 100644 src/libwatcher.jl diff --git a/src/BetterFileWatching.jl b/src/BetterFileWatching.jl index 644f6d0..95b7d83 100644 --- a/src/BetterFileWatching.jl +++ b/src/BetterFileWatching.jl @@ -1,9 +1,6 @@ module BetterFileWatching -using Deno_jll - -import JSON - +include("./libwatcher.jl") abstract type FileEvent end @@ -18,14 +15,25 @@ struct Created <: FileEvent end struct Accessed <: FileEvent paths::Vector{String} + + function Accessed(p) + @warn "Accessed is deprecated and will be removed in the following versions." + end end -const mapFileEvent = Dict( - "modify" => Modified, - "create" => Created, - "remove" => Removed, - "access" => Accessed, -) +function convert_to_deno_events(events::Vector{Event}) + reduce(events; init=(; modified=Modified(String[]), removed=Removed(String[]), created=Created(String[]))) do acc, event + if event.is_created + push!(acc.created.paths, event.path) + elseif event.is_deleted + push!(acc.removed.paths, event.path) + else + push!(acc.modified.paths, event.path) + end + + acc + end +end export watch_folder, watch_file @@ -73,71 +81,25 @@ $(_doc_examples(true)) - `BetterFileWatching.watch_folder` also watching file _contents_ for changes. - BetterFileWatching.jl is based on [Deno.watchFs](https://doc.deno.land/builtin/stable#Deno.watchFs), made available through the [Deno_jll](https://github.com/JuliaBinaryWrappers/Deno_jll.jl) package. """ -function watch_folder(on_event::Function, dir::AbstractString="."; ignore_accessed::Bool=true, ignore_dotgit::Bool=true) - script = """ - const watcher = Deno.watchFs($(JSON.json(dir))); - for await (const event of watcher) { - try { - await Deno.stdout.write(new TextEncoder().encode("\\n" + JSON.stringify(event) + "\\n")); - } catch(e) { - Deno.exit(); - } - } - """ - - outpipe = Pipe() - - function on_stdout(str) - for s in split(str, "\n"; keepempty=false) - local event_raw = nothing - event = try - event_raw = JSON.parse(s) - T = mapFileEvent[event_raw["kind"]] - T(String.(event_raw["paths"])) - catch e - @warn "Unrecognized file watching event. Please report this to https://github.com/JuliaPluto/BetterFileWatching.jl" event_raw ex=(e,catch_backtrace()) - end - if !(ignore_accessed && event isa Accessed) - if !(ignore_dotgit && event isa FileEvent && all(".git" ∈ splitpath(relpath(path, dir)) for path in event.paths)) - on_event(event) - end - end - end +function watch_folder(on_event::Function, dir::AbstractString="."; ignore_accessed::Union{Bool,Nothing}=nothing, ignore_dotgit::Bool=true) + # blocking version with a callback + if ignore_accessed !== nothing + @warn "ignore_accessed is deprecated and will be removed in the coming versions." end - deno_task = @async run(pipeline(`$(deno()) eval $(script)`; stdout=outpipe)) - watch_task = @async try - sleep(.1) - while true - on_stdout(String(readavailable(outpipe))) - end - catch e - if !istaskdone(deno_task) - schedule(deno_task, e; error=true) - end - if !(e isa InterruptException) - showerror(stderr, e, catch_backtrace()) - end + watch(dir) do events + events = convert_to_deno_events(events) + length(events.modified.paths) > 0 && on_event(events.modified) + length(events.created.paths) > 0 && on_event(events.created) + length(events.removed.paths) > 0 && on_event(events.removed) end - - try wait(watch_task) catch; end end function watch_folder(dir::AbstractString="."; kwargs...)::Union{Nothing,FileEvent} - event = Ref{Union{Nothing,FileEvent}}(nothing) - task = Ref{Task}() - task[] = @async watch_folder(dir; kwargs...) do e - event[] = e - try - schedule(task[], InterruptException(); error=true) - catch; end - end - wait(task[]) - event[] + # blocking without callback end - """ ```julia watch_file(f::Function, filename::AbstractString) diff --git a/src/libwatcher.jl b/src/libwatcher.jl new file mode 100644 index 0000000..7aabeb4 --- /dev/null +++ b/src/libwatcher.jl @@ -0,0 +1,201 @@ +libwatcher = "/home/paul/Projects/watcher/zig-out/lib/libwatcher.so" + +mutable struct Watcher + cond::Base.AsyncCondition + watcher::Ptr{Nothing} + + dir::AbstractString +end +Watcher(f::Function, ptr, dir::AbstractString) = Watcher(Base.AsyncCondition(f), ptr, dir) + +Base.show(io::IO, w::Watcher) = write(io, "Watcher(\"", w.dir, "\")") + +Base.@kwdef struct Options + ignores::Set{String} = Set{String}() + backend = "default" +end + +function watcher_write_snapshot(dir, snapshot_path) + @ccall libwatcher.watcher_write_snapshot(dir::Cstring, snapshot_path::Cstring)::Cvoid +end + +function watcher_get_events_since(dir, snapshot_path, callback) + @ccall libwatcher.watcher_get_events_since(dir::Cstring, snapshot_path::Cstring, callback::Ptr{Nothing})::Cvoid +end + +function watcher_subscribe(dir, handle) + @ccall libwatcher.watcher_subscribe(dir::Cstring, handle::Ptr{Nothing})::Cvoid +end +watcher_subscribe(w::Watcher) = watcher_subscribe(w.dir, w.cond.handle) + +function watcher_unsubscribe(dir) + @ccall libwatcher.watcher_unsubscribe(dir::Cstring)::Cvoid +end +watcher_unsubscribe(w::Watcher) = watcher_unsubscribe(w.dir) + +function watcher_get_watcher(dir, options) + @ccall libwatcher.watcher_get_watcher(dir::Cstring, options::Ptr{Nothing})::Ptr{Nothing} +end + +function watcher_watcher_get_events(watcher, events) + @ccall libwatcher.watcher_watcher_get_events(watcher::Ptr{Nothing}, events::Ptr{Nothing})::Ptr{Nothing} +end + +function to_options_ptr(options) + options_ptr = @ccall libwatcher.watcher_new_options()::Ptr{Nothing} + for ignore in options.ignores + @ccall libwatcher.watcher_options_add_ignore(options_ptr::Ptr{Nothing}, ignore::Cstring)::Ptr{Nothing} + end + @ccall libwatcher.watcher_options_set_backend(options_ptr::Ptr{Nothing}, options.backend::Cstring)::Ptr{Nothing} + + options_ptr +end + +function watcher_delete_options(options) + @ccall libwatcher.watcher_delete_options(options::Ptr{Nothing})::Cvoid +end + +struct JLEvent + path::Ptr{Int8} + path_length::Csize_t + is_created::Bool + is_deleted::Bool +end + +struct Event + path::String + is_created::Bool + is_deleted::Bool +end +Event(jl_event::JLEvent) = Event( + Base.unsafe_string(jl_event.path, jl_event.path_length), + jl_event.is_created, + jl_event.is_deleted +) + +const global_watchers = Dict{String,Watcher}() + +function _subscribe_callback(f) + function _c_callback(cevents::Ptr{JLEvent}, n_events::Csize_t) + events = Event[] + + if cevents == C_NULL + f(Event[]) + return nothing + end + + for i in 0:n_events-1 + jl_event = Base.unsafe_load(cevents + i * sizeof(JLEvent)) + if jl_event.path_length == 0 + @warn "got path of length 0" + continue + end + push!(events, Event(jl_event)) + end + + @async f(events) + + nothing + end + + @cfunction($_c_callback, Cvoid, (Ptr{JLEvent}, Csize_t)) +end + +function sanitize_path(path) + length(path) == 0 && error("path can't be empty") + path # TODO +end + +mutable struct Events + size::Csize_t + events::Ptr{JLEvent} +end +Events() = Events(0, Ptr{JLEvent}()) + +function _get_events(watcher_ptr) + events = Events() + watcher_watcher_get_events(watcher_ptr, Base.pointer_from_objref(events)) + events.events == C_NULL && error("Failed to fetch events from watcher.") + + Base.unsafe_wrap(Array, events.events, events.size; own=true) +end + +function subscribe(f::Function, dir, options=Options()) + dir = sanitize_path(dir) + + options_ptr = to_options_ptr(options) + watcher_ptr = watcher_get_watcher(dir, options_ptr); + watcher_delete_options(options_ptr) + + function callback(_) + try + events = Event.(_get_events(watcher_ptr)) + f(events) + catch err + @error "something went wrong" err + end + end + + cond = Base.AsyncCondition(callback) + watcher = Watcher(cond, watcher_ptr, dir) + + watcher_subscribe(watcher) + + watcher +end + +function unsubscribe(watcher) + watcher_unsubscribe(watcher) + + delete!(global_watchers, watcher.dir) + nothing +end + +function write_snapshot(dir, snapshot_path) + dir = sanitize_path(dir) + + watcher_write_snapshot(dir, snapshot_path); +end + +function get_events_since(dir, snapshot) + dir = sanitize_path(dir) + + events_ref = Ref{Vector{Event}}(Event[]) + callback = _subscribe_callback() do events + events_ref[] = events + end + + GC.@preserve callback watcher_get_events_since(dir, snapshot, callback) + + events_ref[] +end + +"Synchronous API" +function watch(f::Function, dir::AbstractString, options=Options()) + dir = sanitize_path(dir) + + options_ptr = to_options_ptr(options) + watcher_ptr = watcher_get_watcher(dir, options_ptr) + watcher_delete_options(options_ptr) + + cond = Base.AsyncCondition() + w = Watcher(cond, watcher_ptr, dir) + + task = Task() do + try + while true + wait(cond) + + events = Event.(_get_events(watcher_ptr)) + f(events) + end + finally + unsubscribe(w) + end + end + + watcher_subscribe(w) + + schedule(task) + wait(task) +end From bfc1a8ce950297fb35932e4925479707d455d2b6 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 17 Jan 2022 20:24:13 +0100 Subject: [PATCH 2/8] =?UTF-8?q?legacy=20tests=20pass=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/BetterFileWatching.jl | 19 +++ src/libwatcher.jl | 265 +++++++++++++++++++++----------------- test/runtests.jl | 5 +- 3 files changed, 169 insertions(+), 120 deletions(-) diff --git a/src/BetterFileWatching.jl b/src/BetterFileWatching.jl index 95b7d83..19fb917 100644 --- a/src/BetterFileWatching.jl +++ b/src/BetterFileWatching.jl @@ -97,7 +97,26 @@ end function watch_folder(dir::AbstractString="."; kwargs...)::Union{Nothing,FileEvent} + # legacy API ----- # blocking without callback + chan = Channel{FileEvent}(1) + + watcher = subscribe(dir) do events + events = convert_to_deno_events(events) + + if length(events.modified.paths) > 0 + put!(chan, events.modified) + elseif length(events.created.paths) > 0 + put!(chan, events.created) + elseif length(events.removed.paths) > 0 + put!(chan, events.removed) + end + end + + event = take!(chan) + unsubscribe(watcher) + + event end """ diff --git a/src/libwatcher.jl b/src/libwatcher.jl index 7aabeb4..6430cba 100644 --- a/src/libwatcher.jl +++ b/src/libwatcher.jl @@ -1,201 +1,230 @@ -libwatcher = "/home/paul/Projects/watcher/zig-out/lib/libwatcher.so" +libwatcher = "/home/paul/Projects/watcher/build/libwatcher.so" mutable struct Watcher - cond::Base.AsyncCondition - watcher::Ptr{Nothing} + cond::Base.AsyncCondition + chan::Channel{Nothing} + watcher::Ptr{Nothing} - dir::AbstractString + dir::String + + Watcher(cond::Base.AsyncCondition, chan::Channel{Nothing}, watcher::Ptr{Nothing}, dir::AbstractString) = + finalizer(watcher_unsubscribe, new(cond, chan, watcher, dir)) end Watcher(f::Function, ptr, dir::AbstractString) = Watcher(Base.AsyncCondition(f), ptr, dir) Base.show(io::IO, w::Watcher) = write(io, "Watcher(\"", w.dir, "\")") Base.@kwdef struct Options - ignores::Set{String} = Set{String}() - backend = "default" + ignores::Set{String} = Set{String}() + backend = "default" end function watcher_write_snapshot(dir, snapshot_path) - @ccall libwatcher.watcher_write_snapshot(dir::Cstring, snapshot_path::Cstring)::Cvoid + @ccall libwatcher.watcher_write_snapshot(dir::Cstring, snapshot_path::Cstring)::Cvoid end -function watcher_get_events_since(dir, snapshot_path, callback) - @ccall libwatcher.watcher_get_events_since(dir::Cstring, snapshot_path::Cstring, callback::Ptr{Nothing})::Cvoid +function watcher_get_events_since(dir, snapshot_path, events) + @ccall libwatcher.watcher_get_events_since( + dir::Cstring, + snapshot_path::Cstring, + events::Ptr{Nothing}, + )::Cvoid end function watcher_subscribe(dir, handle) - @ccall libwatcher.watcher_subscribe(dir::Cstring, handle::Ptr{Nothing})::Cvoid + @ccall libwatcher.watcher_subscribe(dir::Cstring, handle::Ptr{Nothing})::Cvoid end watcher_subscribe(w::Watcher) = watcher_subscribe(w.dir, w.cond.handle) function watcher_unsubscribe(dir) - @ccall libwatcher.watcher_unsubscribe(dir::Cstring)::Cvoid + @ccall libwatcher.watcher_unsubscribe(dir::Cstring)::Cvoid +end + +function watcher_unsubscribe(watcher::Watcher) + if isopen(watcher.cond) + take!(watcher.chan) + isopen(watcher.cond) || return + + watcher_unsubscribe(watcher.dir) # FIXME: use watcher instead of dir + Base.close(watcher.cond) + end + + nothing end -watcher_unsubscribe(w::Watcher) = watcher_unsubscribe(w.dir) function watcher_get_watcher(dir, options) - @ccall libwatcher.watcher_get_watcher(dir::Cstring, options::Ptr{Nothing})::Ptr{Nothing} + @ccall libwatcher.watcher_get_watcher(dir::Cstring, options::Ptr{Nothing})::Ptr{Nothing} +end + +function watcher_delete_watcher(watcher) + @ccall libwatcher.watcher_delete_watcher(watcher::Ptr{Nothing})::Cvoid end function watcher_watcher_get_events(watcher, events) - @ccall libwatcher.watcher_watcher_get_events(watcher::Ptr{Nothing}, events::Ptr{Nothing})::Ptr{Nothing} + @ccall libwatcher.watcher_watcher_get_events( + watcher::Ptr{Nothing}, + events::Ptr{Nothing}, + )::Ptr{Nothing} end function to_options_ptr(options) - options_ptr = @ccall libwatcher.watcher_new_options()::Ptr{Nothing} - for ignore in options.ignores - @ccall libwatcher.watcher_options_add_ignore(options_ptr::Ptr{Nothing}, ignore::Cstring)::Ptr{Nothing} - end - @ccall libwatcher.watcher_options_set_backend(options_ptr::Ptr{Nothing}, options.backend::Cstring)::Ptr{Nothing} + options_ptr = @ccall libwatcher.watcher_new_options()::Ptr{Nothing} + for ignore in options.ignores + @ccall libwatcher.watcher_options_add_ignore( + options_ptr::Ptr{Nothing}, + ignore::Cstring, + )::Ptr{Nothing} + end + @ccall libwatcher.watcher_options_set_backend( + options_ptr::Ptr{Nothing}, + options.backend::Cstring, + )::Ptr{Nothing} - options_ptr + options_ptr end function watcher_delete_options(options) - @ccall libwatcher.watcher_delete_options(options::Ptr{Nothing})::Cvoid + @ccall libwatcher.watcher_delete_options(options::Ptr{Nothing})::Cvoid end struct JLEvent - path::Ptr{Int8} - path_length::Csize_t - is_created::Bool - is_deleted::Bool + path::Ptr{Cchar} + path_length::Csize_t + is_created::Bool + is_deleted::Bool end struct Event - path::String - is_created::Bool - is_deleted::Bool + path::String + is_created::Bool + is_deleted::Bool end Event(jl_event::JLEvent) = Event( - Base.unsafe_string(jl_event.path, jl_event.path_length), - jl_event.is_created, - jl_event.is_deleted + Base.unsafe_string(jl_event.path, jl_event.path_length), + jl_event.is_created, + jl_event.is_deleted, ) -const global_watchers = Dict{String,Watcher}() - -function _subscribe_callback(f) - function _c_callback(cevents::Ptr{JLEvent}, n_events::Csize_t) - events = Event[] - - if cevents == C_NULL - f(Event[]) - return nothing - end - - for i in 0:n_events-1 - jl_event = Base.unsafe_load(cevents + i * sizeof(JLEvent)) - if jl_event.path_length == 0 - @warn "got path of length 0" - continue - end - push!(events, Event(jl_event)) - end - - @async f(events) - - nothing - end - - @cfunction($_c_callback, Cvoid, (Ptr{JLEvent}, Csize_t)) -end - function sanitize_path(path) - length(path) == 0 && error("path can't be empty") - path # TODO + length(path) == 0 && error("path can't be empty") + path # TODO end mutable struct Events - size::Csize_t - events::Ptr{JLEvent} + size::Csize_t + events::Ptr{JLEvent} end Events() = Events(0, Ptr{JLEvent}()) function _get_events(watcher_ptr) - events = Events() - watcher_watcher_get_events(watcher_ptr, Base.pointer_from_objref(events)) - events.events == C_NULL && error("Failed to fetch events from watcher.") - - Base.unsafe_wrap(Array, events.events, events.size; own=true) + events = Events() + GC.@preserve events watcher_watcher_get_events( + watcher_ptr, + Base.pointer_from_objref(events), + ) + events.events == C_NULL && error("Failed to fetch events from watcher.") + + raw_events = Base.unsafe_wrap(Array, events.events, events.size; own = true) + + map(raw_events) do raw_event + event = Event(raw_event) + ccall(:free, Cvoid, (Ptr{Nothing},), raw_event.path) + event + end end -function subscribe(f::Function, dir, options=Options()) - dir = sanitize_path(dir) +function subscribe(f::Function, dir, options = Options()) + dir = sanitize_path(dir) - options_ptr = to_options_ptr(options) - watcher_ptr = watcher_get_watcher(dir, options_ptr); - watcher_delete_options(options_ptr) + options_ptr = to_options_ptr(options) + watcher_ptr = watcher_get_watcher(dir, options_ptr) + watcher_delete_options(options_ptr) - function callback(_) - try - events = Event.(_get_events(watcher_ptr)) - f(events) - catch err - @error "something went wrong" err + chan = Channel{Nothing}(1) + put!(chan, nothing) + + function callback(_) + try + take!(chan) + events = _get_events(watcher_ptr) + put!(chan, nothing) + + f(events) + catch err + @error "something went wrong" err + end end - end - cond = Base.AsyncCondition(callback) - watcher = Watcher(cond, watcher_ptr, dir) + cond = Base.AsyncCondition(callback) + watcher = Watcher(cond, chan, watcher_ptr, dir) - watcher_subscribe(watcher) + watcher_subscribe(watcher) - watcher + watcher end function unsubscribe(watcher) - watcher_unsubscribe(watcher) + watcher_unsubscribe(watcher) - delete!(global_watchers, watcher.dir) - nothing + nothing end function write_snapshot(dir, snapshot_path) - dir = sanitize_path(dir) + dir = sanitize_path(dir) - watcher_write_snapshot(dir, snapshot_path); + watcher_write_snapshot(dir, snapshot_path) end function get_events_since(dir, snapshot) - dir = sanitize_path(dir) + dir = sanitize_path(dir) - events_ref = Ref{Vector{Event}}(Event[]) - callback = _subscribe_callback() do events - events_ref[] = events - end + events = Events() + GC.@preserve events watcher_get_events_since( + dir, + snapshot, + Base.pointer_from_objref(events), + ) + events.events == C_NULL && error("Failed to get events since for '$snapshot'") - GC.@preserve callback watcher_get_events_since(dir, snapshot, callback) - - events_ref[] + Base.unsafe_wrap(Array, events.events, events.size; own = true) end "Synchronous API" -function watch(f::Function, dir::AbstractString, options=Options()) - dir = sanitize_path(dir) +function watch(f::Function, dir::AbstractString, options = Options()) + dir = sanitize_path(dir) + + options_ptr = to_options_ptr(options) + watcher_ptr = watcher_get_watcher(dir, options_ptr) + watcher_delete_options(options_ptr) + + cond = Base.AsyncCondition() + chan = Channel{Nothing}(1) + put!(chan, nothing) + w = Watcher(cond, chan, watcher_ptr, dir) + + task = Task() do + try + while isopen(cond) + wait(cond) + + take!(chan) + events = _get_events(watcher_ptr) + put!(chan, nothing) + f(events) + end + finally + unsubscribe(w) + end + end - options_ptr = to_options_ptr(options) - watcher_ptr = watcher_get_watcher(dir, options_ptr) - watcher_delete_options(options_ptr) + watcher_subscribe(w) - cond = Base.AsyncCondition() - w = Watcher(cond, watcher_ptr, dir) + schedule(task) - task = Task() do try - while true - wait(cond) - - events = Event.(_get_events(watcher_ptr)) - f(events) - end - finally - unsubscribe(w) + wait(task) + catch end - end - - watcher_subscribe(w) - schedule(task) - wait(task) + nothing end diff --git a/test/runtests.jl b/test/runtests.jl index 077f73a..4c6d05c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -55,11 +55,13 @@ end push!(events, e) end else - @asynclog while true + @info "using legacy" + t = @async while true e = watch_folder(test_dir) @info "Event" e push!(events, e) end + t end sleep(2) @@ -98,4 +100,3 @@ end sleep(2) @test_nowarn schedule(watch_task, InterruptException(); error=true) end - From d8a5e51ded1ebea50d094c45595b66574c77b87e Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 17 Feb 2022 22:54:29 +0100 Subject: [PATCH 3/8] use the watcher handle --- src/BetterFileWatching.jl | 48 ++++++++-- src/libwatcher.jl | 187 ++++++++++++++++++++++++++++---------- test/runtests.jl | 2 +- test/snapshots.jl | 55 +++++++++++ 4 files changed, 236 insertions(+), 56 deletions(-) create mode 100644 test/snapshots.jl diff --git a/src/BetterFileWatching.jl b/src/BetterFileWatching.jl index 19fb917..c3346f1 100644 --- a/src/BetterFileWatching.jl +++ b/src/BetterFileWatching.jl @@ -79,15 +79,15 @@ $(_doc_examples(true)) - `BetterFileWatching.watch_folder` works _recursively_, i.e. subfolders are also watched. - `BetterFileWatching.watch_folder` also watching file _contents_ for changes. -- BetterFileWatching.jl is based on [Deno.watchFs](https://doc.deno.land/builtin/stable#Deno.watchFs), made available through the [Deno_jll](https://github.com/JuliaBinaryWrappers/Deno_jll.jl) package. +- BetterFileWatching.jl is based on a port of [parcel-bundler/watcher](https://github.com/parcel-bundler/watcher) to Julia (available on [JuliaPluto/watcher](https://github.com/JuliaPluto/watcher)) """ function watch_folder(on_event::Function, dir::AbstractString="."; ignore_accessed::Union{Bool,Nothing}=nothing, ignore_dotgit::Bool=true) # blocking version with a callback - if ignore_accessed !== nothing + if ignore_accessed === true @warn "ignore_accessed is deprecated and will be removed in the coming versions." end - watch(dir) do events + watch_folder_sync(dir; options = Options(ignores=Set{String}(ignore_dotgit ? [] : [".git/"]))) do events events = convert_to_deno_events(events) length(events.modified.paths) > 0 && on_event(events.modified) length(events.created.paths) > 0 && on_event(events.created) @@ -132,9 +132,45 @@ $(_doc_examples(false)) # Differences with the FileWatching stdlib -- BetterFileWatching.jl is based on [Deno.watchFs](https://doc.deno.land/builtin/stable#Deno.watchFs), made available through the [Deno_jll](https://github.com/JuliaBinaryWrappers/Deno_jll.jl) package. +- BetterFileWatching.jl is based on a port of [parcel-bundler/watcher](https://github.com/parcel-bundler/watcher) to Julia (available on [JuliaPluto/watcher](https://github.com/JuliaPluto/watcher)) """ -watch_file(filename::AbstractString; kwargs...) = watch_folder(filename; kwargs...) -watch_file(f::Function, filename::AbstractString; kwargs...) = watch_folder(f, filename; kwargs...) +function watch_file(filename::AbstractString; kwargs...) + file_path = abspath(filename) + dir_path = dirname(file_path) + + chan = Channel{FileEvent}(1) + + watcher = subscribe(dir_path) do events + file_event = findfirst(event -> abspath(event.path) == file_path, events) + file_event === nothing && return + events = convert_to_deno_events([events[file_event]]) + + if length(events.modified.paths) > 0 + put!(chan, events.modified) + elseif length(events.created.paths) > 0 + put!(chan, events.created) + elseif length(events.removed.paths) > 0 + put!(chan, events.removed) + end + end + + event = take!(chan) + unsubscribe(watcher) + + event +end + +function watch_file(f::Function, filename::AbstractString; kwargs...) + file_path = abspath(filename) + dir_path = dirname(file_path) + watch_folder(dir_path; kwargs...) do event + filter!(event.paths) do path + normpath(path) == file_path + end + if length(event.paths) > 0 + f(event) + end + end +end end diff --git a/src/libwatcher.jl b/src/libwatcher.jl index 6430cba..d60ace4 100644 --- a/src/libwatcher.jl +++ b/src/libwatcher.jl @@ -1,16 +1,37 @@ libwatcher = "/home/paul/Projects/watcher/build/libwatcher.so" +include_dependency(libwatcher) + +mutable struct VariableSize{N} + data::NTuple{N,UInt8} +end +const handle_size = Ref{Int}(-1) + +function WatcherHandle() + if handle_size[] == -1 + handle_size[] = @ccall libwatcher.watcher_watcher_handle_sizeof()::Csize_t + end + + s = handle_size[] + + VariableSize{s}(Tuple(0 for _ = 1:s)) +end mutable struct Watcher cond::Base.AsyncCondition chan::Channel{Nothing} - watcher::Ptr{Nothing} + handle::VariableSize dir::String - Watcher(cond::Base.AsyncCondition, chan::Channel{Nothing}, watcher::Ptr{Nothing}, dir::AbstractString) = - finalizer(watcher_unsubscribe, new(cond, chan, watcher, dir)) + Watcher( + cond::Base.AsyncCondition, + chan::Channel{Nothing}, + handle::VariableSize, + dir::AbstractString, + ) = finalizer(watcher_unsubscribe, new(cond, chan, handle, dir)) end -Watcher(f::Function, ptr, dir::AbstractString) = Watcher(Base.AsyncCondition(f), ptr, dir) +Watcher(f::Function, handle, dir::AbstractString) = + Watcher(Base.AsyncCondition(f), handle, dir) Base.show(io::IO, w::Watcher) = write(io, "Watcher(\"", w.dir, "\")") @@ -19,25 +40,44 @@ Base.@kwdef struct Options backend = "default" end -function watcher_write_snapshot(dir, snapshot_path) - @ccall libwatcher.watcher_write_snapshot(dir::Cstring, snapshot_path::Cstring)::Cvoid +function watcher_write_snapshot(dir, snapshot_path, options) + @ccall libwatcher.watcher_write_snapshot( + dir::Cstring, + snapshot_path::Cstring, + options::Ptr{Nothing}, + )::Cvoid end -function watcher_get_events_since(dir, snapshot_path, events) +function watcher_get_events_since(dir, snapshot_path, events, options) @ccall libwatcher.watcher_get_events_since( dir::Cstring, snapshot_path::Cstring, events::Ptr{Nothing}, + options::Ptr{Nothing}, )::Cvoid end -function watcher_subscribe(dir, handle) - @ccall libwatcher.watcher_subscribe(dir::Cstring, handle::Ptr{Nothing})::Cvoid +function watcher_subscribe(dir, handle, options, watcher_handle) + @ccall libwatcher.watcher_subscribe( + dir::Cstring, + handle::Ptr{Nothing}, + options::Ptr{Nothing}, + watcher_handle::Ptr{Nothing}, + )::Cvoid +end +function watcher_subscribe(w::Watcher, options) + GC.@preserve w watcher_subscribe( + w.dir, + w.cond.handle, + options, + Base.pointer_from_objref(w.handle), + ) end -watcher_subscribe(w::Watcher) = watcher_subscribe(w.dir, w.cond.handle) -function watcher_unsubscribe(dir) - @ccall libwatcher.watcher_unsubscribe(dir::Cstring)::Cvoid +function watcher_unsubscribe(handle) + GC.@preserve handle @ccall libwatcher.watcher_unsubscribe( + Base.pointer_from_objref(handle)::Ptr{Cvoid}, + )::Cvoid end function watcher_unsubscribe(watcher::Watcher) @@ -45,7 +85,7 @@ function watcher_unsubscribe(watcher::Watcher) take!(watcher.chan) isopen(watcher.cond) || return - watcher_unsubscribe(watcher.dir) # FIXME: use watcher instead of dir + watcher_unsubscribe(watcher.handle) Base.close(watcher.cond) end @@ -68,6 +108,11 @@ function watcher_watcher_get_events(watcher, events) end function to_options_ptr(options) + options = Options( + ignores = Set{String}(abspath(p) for p in options.ignores), + backend = options.backend, + ) + options_ptr = @ccall libwatcher.watcher_new_options()::Ptr{Nothing} for ignore in options.ignores @ccall libwatcher.watcher_options_add_ignore( @@ -75,7 +120,7 @@ function to_options_ptr(options) ignore::Cstring, )::Ptr{Nothing} end - @ccall libwatcher.watcher_options_set_backend( + GC.@preserve options @ccall libwatcher.watcher_options_set_backend( options_ptr::Ptr{Nothing}, options.backend::Cstring, )::Ptr{Nothing} @@ -87,11 +132,15 @@ function watcher_delete_options(options) @ccall libwatcher.watcher_delete_options(options::Ptr{Nothing})::Cvoid end +function watcher_delete_events(events) + @ccall libwatcher.watcher_delete_events(events::Ptr{Nothing})::Cvoid +end + struct JLEvent path::Ptr{Cchar} path_length::Csize_t - is_created::Bool - is_deleted::Bool + is_created::Cuchar + is_deleted::Cuchar end struct Event @@ -107,38 +156,41 @@ Event(jl_event::JLEvent) = Event( function sanitize_path(path) length(path) == 0 && error("path can't be empty") - path # TODO + isdir(path) || error("path $path should be a directory") + abspath(path) end mutable struct Events size::Csize_t events::Ptr{JLEvent} + + Events(size, events) = finalizer( + e -> begin + watcher_delete_events(Base.pointer_from_objref(e)) + end, + new(size, events), + ) end Events() = Events(0, Ptr{JLEvent}()) function _get_events(watcher_ptr) events = Events() - GC.@preserve events watcher_watcher_get_events( - watcher_ptr, + GC.@preserve events watcher_ptr watcher_watcher_get_events( + Base.pointer_from_objref(watcher_ptr), Base.pointer_from_objref(events), ) events.events == C_NULL && error("Failed to fetch events from watcher.") - raw_events = Base.unsafe_wrap(Array, events.events, events.size; own = true) - - map(raw_events) do raw_event - event = Event(raw_event) - ccall(:free, Cvoid, (Ptr{Nothing},), raw_event.path) - event + GC.@preserve events begin + raw_events = Base.unsafe_wrap(Array, events.events, events.size) + Event.(raw_events) end end function subscribe(f::Function, dir, options = Options()) dir = sanitize_path(dir) - options_ptr = to_options_ptr(options) - watcher_ptr = watcher_get_watcher(dir, options_ptr) - watcher_delete_options(options_ptr) + handle = WatcherHandle() chan = Channel{Nothing}(1) put!(chan, nothing) @@ -146,7 +198,7 @@ function subscribe(f::Function, dir, options = Options()) function callback(_) try take!(chan) - events = _get_events(watcher_ptr) + events = _get_events(handle) put!(chan, nothing) f(events) @@ -156,9 +208,13 @@ function subscribe(f::Function, dir, options = Options()) end cond = Base.AsyncCondition(callback) - watcher = Watcher(cond, chan, watcher_ptr, dir) + watcher = Watcher(cond, chan, handle, dir) - watcher_subscribe(watcher) + GC.@preserve options begin + options_ptr = to_options_ptr(options) + watcher_subscribe(watcher, options_ptr) + watcher_delete_options(options_ptr) + end watcher end @@ -169,38 +225,67 @@ function unsubscribe(watcher) nothing end -function write_snapshot(dir, snapshot_path) +function validate_snapshot_path(path) + parent_path = abspath(path) |> dirname + if !isdir(parent_path) + error("Folder $parent_path does not exist for snapshot file $path") + end + length(path) == 0 && error("An empty path is not valid") +end + +""" + write_snapshot(dir::AbstractString, snapshot_path::AbstractString)::Nothing + +Writes a snapshot file to snaphot_path from the directory dir. The written snapshot can +then be used with `get_events_since(dir, snapshot_path)` to retrieve the changes. +""" +function write_snapshot(dir, snapshot_path; options = Options()) dir = sanitize_path(dir) + validate_snapshot_path(snapshot_path) + + options_ptr = to_options_ptr(options) + watcher_write_snapshot(dir, snapshot_path, options_ptr) + watcher_delete_options(options_ptr) - watcher_write_snapshot(dir, snapshot_path) + nothing end -function get_events_since(dir, snapshot) +""" +get_events_since(dir::AbstractString, snapshot_path::AbstractString)::Vector{Event} + +""" +function get_events_since(dir, snapshot_path; options = Options()) dir = sanitize_path(dir) + validate_snapshot_path(snapshot_path) events = Events() - GC.@preserve events watcher_get_events_since( - dir, - snapshot, - Base.pointer_from_objref(events), - ) - events.events == C_NULL && error("Failed to get events since for '$snapshot'") - - Base.unsafe_wrap(Array, events.events, events.size; own = true) + GC.@preserve events options begin + + options_ptr = to_options_ptr(options) + watcher_get_events_since( + dir, + snapshot_path, + Base.pointer_from_objref(events), + options_ptr, + ) + watcher_delete_options(options_ptr) + events.events == C_NULL && error("Failed to get events since for '$snapshot_path'") + + Event.(Base.unsafe_wrap(Array, events.events, events.size)) + end end "Synchronous API" -function watch(f::Function, dir::AbstractString, options = Options()) +function watch_folder_sync(f::Function, dir::AbstractString; options = Options()) dir = sanitize_path(dir) - options_ptr = to_options_ptr(options) - watcher_ptr = watcher_get_watcher(dir, options_ptr) - watcher_delete_options(options_ptr) + watcher_handle = WatcherHandle() + cond = Base.AsyncCondition() chan = Channel{Nothing}(1) put!(chan, nothing) - w = Watcher(cond, chan, watcher_ptr, dir) + w = Watcher(cond, chan, watcher_handle, dir) task = Task() do try @@ -208,7 +293,7 @@ function watch(f::Function, dir::AbstractString, options = Options()) wait(cond) take!(chan) - events = _get_events(watcher_ptr) + events = _get_events(watcher_handle) put!(chan, nothing) f(events) end @@ -217,7 +302,11 @@ function watch(f::Function, dir::AbstractString, options = Options()) end end - watcher_subscribe(w) + GC.@preserve options begin + options_ptr = to_options_ptr(options) + watcher_subscribe(w, options_ptr) + watcher_delete_options(options_ptr) + end schedule(task) diff --git a/test/runtests.jl b/test/runtests.jl index 4c6d05c..488f333 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,7 +29,7 @@ macro asynclog(expr) end end - +include("./snapshots.jl") @testset "Basic - $(method)" for method in ["new", "legacy"] test_dir = tempname(cleanup=false) diff --git a/test/snapshots.jl b/test/snapshots.jl new file mode 100644 index 0000000..a2a5f06 --- /dev/null +++ b/test/snapshots.jl @@ -0,0 +1,55 @@ +@enum Action Modify Create Delete + +@testset "Snapshots" begin + + @testset "write/get" begin + tmp_dir = mktempdir() + + write_file(p, content) = write(joinpath(tmp_dir, p), content) + delete_file(p) = rm(joinpath(tmp_dir, p)) + + write_file("not_modified.txt", "sticky") + write_file("already_existing.txt", "Hello") + write_file("should_be_deleted.txt", "oh no") + + snapshot_file = joinpath(tmp_dir, "snapshot.txt") + BetterFileWatching.write_snapshot(tmp_dir, snapshot_file) + + delete_file("should_be_deleted.txt") + write_file("already_existing.txt", "modified") + write_file("newly_created.txt", "hey!") + + events = BetterFileWatching.get_events_since(tmp_dir, snapshot_file) + + @test count(events) do event + basename(event.path) == "should_be_deleted.txt" && + event.is_deleted && + !event.is_created + end == 1 + + @test count(events) do event + basename(event.path) == "newly_created.txt" && + !event.is_deleted && + event.is_created + end == 1 + + @test count(events) do event + basename(event.path) == "already_existing.txt" && + !event.is_deleted && + !event.is_created + end == 1 + + @test count(events) do event + basename(event.path) == "not_modified.txt" + end == 0 + end + + @testset "Errors" begin + tmp_dir = mktempdir() + @test_throws ErrorException BetterFileWatching.write_snapshot(tmp_dir, "") + + @test_throws ErrorException BetterFileWatching.write_snapshot(tmp_dir, joinpath(tmp_dir, "notafolder/mysnapshot.txt")) + + @test_throws ErrorException BetterFileWatching.write_snapshot("not_existing.folder", "") + end +end From 6c2461ba048bd32098b0dfd8010bb82c002fd0b9 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 17 Feb 2022 22:59:39 +0100 Subject: [PATCH 4/8] readme --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5171338..bad212e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ watch_folder(f::Function, dir=".") Watch a folder recursively for any changes. Includes changes to file contents. A [`FileEvent`](@ref) is passed to the callback function `f`. -# Example +## Examples ```julia watch_folder(".") do event @@ -27,8 +27,22 @@ sleep(5) schedule(watch_task, InterruptException(); error=true) ``` -# Differences with the FileWatching stdlib +## Snapshots + +The library also allow you take snapshots of a directory and read those snapshots later to see exactly which files have been updated/deleted/created. + +```julia +options = BetterFileWatching.Options(ignores = Set{String}(["./.git"])) +BetterFileWatching.write_snapshot(dir, snapshot_path; options = options) + +# Create some, do some changes, delete some files... + +events = BetterFileWatching.get_events_since(dir, snapshot_path; options = options) +``` + +## Differences with the FileWatching stdlib - `BetterFileWatching.watch_folder` works _recursively_, i.e. subfolders are also watched. - `BetterFileWatching.watch_folder` also watching file _contents_ for changes. -- BetterFileWatching.jl is just a small wrapper around [`Deno.watchFs`](https://doc.deno.land/builtin/stable#Deno.watchFs), made available through the [Deno_jll](https://github.com/JuliaBinaryWrappers/Deno_jll.jl) package. `Deno.watchFs` is well-tested and widely used. +- BetterFileWatching.jl is just a wrapper around a port of [parcel-bundler/watcher](https://github.com/parcel-bundler/watcher) to Julia (available on [JuliaPluto/watcher](https://github.com/JuliaPluto/watcher)) + From 1f6b18a942abae8f3607b954067bb746e487817b Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 22 Mar 2022 19:50:53 +0100 Subject: [PATCH 5/8] =?UTF-8?q?use=20libwatcher=5Fjll=20=F0=9F=8E=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project.toml | 2 +- src/libwatcher.jl | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 33193e6..96484ad 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.1.4" [deps] Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +libwatcher_jll = "f2e3f4ed-6d30-53f6-bb33-6a6ab454fa9c" [compat] Deno_jll = "^1.10" @@ -17,4 +18,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test"] - diff --git a/src/libwatcher.jl b/src/libwatcher.jl index d60ace4..dbb0575 100644 --- a/src/libwatcher.jl +++ b/src/libwatcher.jl @@ -1,5 +1,4 @@ -libwatcher = "/home/paul/Projects/watcher/build/libwatcher.so" -include_dependency(libwatcher) +using libwatcher_jll mutable struct VariableSize{N} data::NTuple{N,UInt8} From af4ae5833297810f0aa79e3e0cbdd3164ec1b73a Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 22 Mar 2022 19:57:03 +0100 Subject: [PATCH 6/8] remove Deno and Json --- Project.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Project.toml b/Project.toml index a1100f8..c90f217 100644 --- a/Project.toml +++ b/Project.toml @@ -4,13 +4,9 @@ authors = ["Fons van der Plas "] version = "0.1.5" [deps] -Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d" -JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" libwatcher_jll = "f2e3f4ed-6d30-53f6-bb33-6a6ab454fa9c" [compat] -Deno_jll = "^1.10" -JSON = "^0.20, ^0.21" julia = "1" [extras] From 5041d07d3c0f82374b5f1c3dc3ba8ff0e580aa35 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 22 Mar 2022 19:57:54 +0100 Subject: [PATCH 7/8] tiny --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35efd6c..7b843f3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ watch_folder(f::Function, dir=".") Watch a folder recursively for any changes. Includes changes to file contents. A [`FileEvent`](@ref) is passed to the callback function `f`. ## Examples -======= + ```julia watch_file(f::Function, filename=".") ``` From 7c973b08f5717459d2cb4fa9eb03cc5297eedd3d Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 22 Mar 2022 20:19:06 +0100 Subject: [PATCH 8/8] more docs --- .github/workflows/Test.yml | 2 +- README.md | 8 ++++---- src/BetterFileWatching.jl | 1 + src/libwatcher.jl | 20 +++++++++++++++++++- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index d131458..86fcdc0 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -18,7 +18,7 @@ jobs: # continue-on-error: true strategy: matrix: - julia-version: ["1.6"] + julia-version: ["1.6", "1.7"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: diff --git a/README.md b/README.md index 7b843f3..e98893b 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,12 @@ schedule(watch_task, InterruptException(); error=true) The library also allow you take snapshots of a directory and read those snapshots later to see exactly which files have been updated/deleted/created. ```julia -options = BetterFileWatching.Options(ignores = Set{String}(["./.git"])) -BetterFileWatching.write_snapshot(dir, snapshot_path; options = options) +options = Options(ignores = Set{String}(["./.git"])) +write_snapshot(dir, snapshot_path; options = options) -# Create some, do some changes, delete some files... +# Create some files, do some changes;, delete some files... -events = BetterFileWatching.get_events_since(dir, snapshot_path; options = options) +events = get_events_since(dir, snapshot_path; options = options) ``` ## Differences with the FileWatching stdlib diff --git a/src/BetterFileWatching.jl b/src/BetterFileWatching.jl index 3cf7121..c398082 100644 --- a/src/BetterFileWatching.jl +++ b/src/BetterFileWatching.jl @@ -36,6 +36,7 @@ function convert_to_deno_events(events::Vector{Event}) end export watch_folder, watch_file +export Options, write_snapshot, get_events_since function _doc_examples(folder) f = folder ? "folder" : "file" diff --git a/src/libwatcher.jl b/src/libwatcher.jl index dbb0575..9d57470 100644 --- a/src/libwatcher.jl +++ b/src/libwatcher.jl @@ -39,6 +39,18 @@ Base.@kwdef struct Options backend = "default" end +@doc """ + Options(; ignores::Set{String} = Set{String}(), backend = "default") + +The ignores field of options can be used to prevent receiving file update events when watching a folder. + +```julia +julia> watch_folder_sync("./"; options = Options(ignores = Set(["./git"]))) do events + @info "We have received a batch of events" events + end +``` +""" Options + function watcher_write_snapshot(dir, snapshot_path, options) @ccall libwatcher.watcher_write_snapshot( dir::Cstring, @@ -142,6 +154,10 @@ struct JLEvent is_deleted::Cuchar end +""" +A small struct representing a single file event. If both `event.is_created` and `event.is_deleted` are false then the event corresponds to +a file content change. +""" struct Event path::String is_created::Bool @@ -250,8 +266,10 @@ function write_snapshot(dir, snapshot_path; options = Options()) end """ -get_events_since(dir::AbstractString, snapshot_path::AbstractString)::Vector{Event} + get_events_since(dir::AbstractString, snapshot_path::AbstractString; options = Options())::Vector{Event} +Reads from the snapshot file created using `BetterFileWatching.write_snapshot` and returns the file events +corresponding to changes, creations and deletions since that snapshot was written to. """ function get_events_since(dir, snapshot_path; options = Options()) dir = sanitize_path(dir)