Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add inotify support #168

Merged
merged 7 commits into from
Sep 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
supercronic
vendor
dist/*
.idea
21 changes: 21 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,27 @@ docker kill --signal=USR2 <container id>
kill -USR2 <pid>
```

If you are running Supercronic in an environment were sending `SIGUSR2` is a bit of a hassle, or you expect frequent updates to your crontab file, you may opt to run Supercronic with the `-inotify` flag. This will start a watch on the crontab file, reloading it on changes. An example use case would be a kubernetes pod runnning Supercronic that mounts its crontab file from a configMap. With the `-inotify` flag, any update to this configmap, provided it is not immutable, will trigger a reload in Supercronic, without you having to figure out a mechanism to send the `SIGUSR2` signal to the pod. The watch on the crontab file triggers on `Write` and `Remove` events, the latter ensures detection of kubernetes' atomic writes.

```
$ ./supercronic -inotify ./my-crontab
...
time="2024-09-11T09:23:18+02:00" level=debug msg="event: CHMOD \"./my-crontab\", watch-list: []"
time="2024-09-11T09:23:18+02:00" level=debug msg="event: REMOVE \"./my-crontab\", watch-list: []"
time="2024-09-11T09:23:18+02:00" level=debug msg="watched file changed"
time="2024-09-11T09:23:18+02:00" level=info msg="received user defined signal 2, reloading crontab"
time="2024-09-11T09:23:18+02:00" level=info msg="waiting for jobs to finish"
time="2024-09-11T09:23:18+02:00" level=debug msg="shutting down" job.command="sleep 2" job.position=0 job.schedule="* * * * *"
time="2024-09-11T09:23:18+02:00" level=info msg="read crontab: ./my-crontab"
time="2024-09-11T09:23:18+02:00" level=debug msg="try parse (7 fields): '* * * * * sleep 5'"
time="2024-09-11T09:23:18+02:00" level=debug msg="failed to parse (7 fields): '* * * * * sleep 5': failed: syntax error in day-of-week field: 'sleep'"
time="2024-09-11T09:23:18+02:00" level=debug msg="try parse (6 fields): '* * * * * sleep'"
time="2024-09-11T09:23:18+02:00" level=debug msg="failed to parse (6 fields): '* * * * * sleep': failed: syntax error in year field: 'sleep'"
time="2024-09-11T09:23:18+02:00" level=debug msg="try parse (5 fields): '* * * * *'"
time="2024-09-11T09:23:18+02:00" level=debug msg="job will run next at 2024-09-11 09:24:00 +0200 CEST" job.command="sleep 5" job.position=0 job.schedule="* * * * *"

```

## Testing your crontab

Use the `-test` flag to prompt Supercronic to verify your crontab, but not
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.23.0

require (
github.com/evalphobia/logrus_sentry v0.8.2
github.com/fsnotify/fsnotify v1.7.0
github.com/prometheus/client_golang v1.20.2
github.com/ramr/go-reaper v0.2.1
github.com/sirupsen/logrus v1.9.3
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ=
github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
Expand Down
43 changes: 43 additions & 0 deletions integration/reload.bats
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,46 @@ grep_test_file() {
kill -s TERM "$PID"
wait
}

@test "if inotify is enabled it reloads on file change when receiving a WRITE event" {
echo '* * * * * * * echo a > "$TEST_FILE"' > "$CRONTAB_FILE"

"${BATS_TEST_DIRNAME}/../supercronic" -inotify "$CRONTAB_FILE" 3>&- &
PID="$!"

wait_for grep_test_file a

echo '* * * * * * * echo b > "$TEST_FILE"' > "$CRONTAB_FILE"

wait_for grep_test_file b

kill -s TERM "$PID"
wait
}

@test "if inotify is enabled it handles kubernetes like atomic writes using updated symlinks and folder deletion" {
CRONTAB_FILE_NAME="$(basename $(mktemp --dry-run --tmpdir))"

WORK_DIR="$(mktemp -d)"
CRONTAB_PRE_DIR="$(mktemp -d)"
CRONTAB_POST_DIR="$(mktemp -d)"

echo '* * * * * * * echo a > "$TEST_FILE"' > "$CRONTAB_PRE_DIR"/"$CRONTAB_FILE_NAME"
echo '* * * * * * * echo b > "$TEST_FILE"' > "$CRONTAB_POST_DIR"/"$CRONTAB_FILE_NAME"

ln -s "$CRONTAB_PRE_DIR"/"$CRONTAB_FILE_NAME" "$WORK_DIR"/"$CRONTAB_FILE_NAME"

"${BATS_TEST_DIRNAME}/../supercronic" -inotify -debug "$WORK_DIR"/"$CRONTAB_FILE_NAME" 3>&- &
PID="$!"

wait_for grep_test_file a

ln -sf "$CRONTAB_POST_DIR"/"$CRONTAB_FILE_NAME" "$WORK_DIR"/"$CRONTAB_FILE_NAME"

rm -r "$CRONTAB_PRE_DIR"

wait_for grep_test_file b

kill -s TERM "$PID"
wait
}
Empty file modified integration/test.bats
100644 → 100755
Empty file.
61 changes: 58 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/aptible/supercronic/log/hook"
"github.com/aptible/supercronic/prometheus_metrics"
"github.com/evalphobia/logrus_sentry"
"github.com/fsnotify/fsnotify"
reaper "github.com/ramr/go-reaper"
"github.com/sirupsen/logrus"
)
Expand All @@ -29,6 +30,7 @@ func main() {
quiet := flag.Bool("quiet", false, "do not log informational messages (takes precedence over debug)")
json := flag.Bool("json", false, "enable JSON logging")
test := flag.Bool("test", false, "test crontab (does not run jobs)")
inotify := flag.Bool("inotify", false, "use inotify to detect crontab file changes")
prometheusListen := flag.String(
"prometheus-listen-address",
"",
Expand Down Expand Up @@ -102,6 +104,24 @@ func main() {

crontabFileName := flag.Args()[0]

var watcher *fsnotify.Watcher
if *inotify {
logrus.Info("using inotify to detect crontab file changes")
var err error
watcher, err = fsnotify.NewWatcher()
if err != nil {
logrus.Fatal(err)
return
}
defer watcher.Close()

logrus.Infof("adding file watch for '%s'", crontabFileName)
if err := watcher.Add(crontabFileName); err != nil {
logrus.Fatal(err)
return
}
}

var sentryHook *logrus_sentry.SentryHook
if sentryDsn != "" {
sentryLevels := []logrus.Level{
Expand Down Expand Up @@ -149,6 +169,44 @@ func main() {
go reaper.Reap()
// _ = reaper.Reap

termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR2)

if *inotify {
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
logrus.Debugf("event: %v, watch-list: %v", event, watcher.WatchList())

switch event.Op {
case event.Op & fsnotify.Write:
logrus.Debug("watched file changed")
termChan <- syscall.SIGUSR2

// workaround for k8s configmap and secret mounts
case event.Op & fsnotify.Remove:
logrus.Debug("watched file changed")
if err := watcher.Add(crontabFileName); err != nil {
logrus.Fatal(err)
return
}
termChan <- syscall.SIGUSR2
}

case err, ok := <-watcher.Errors:
if !ok {
return
}
logrus.Error("error:", err)
}
}
}()
}

for {
promMetrics.Reset()

Expand Down Expand Up @@ -179,9 +237,6 @@ func main() {
cron.StartJob(&wg, tab.Context, job, exitCtx, cronLogger, *overlapping, *passthroughLogs, &promMetrics)
}

termChan := make(chan os.Signal, 1)
signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR2)

termSig := <-termChan

if termSig == syscall.SIGUSR2 {
Expand Down
Loading