From 88cbd3030071ec42f1ebd379869bea3821f386a8 Mon Sep 17 00:00:00 2001 From: Stefan Schwarz Date: Mon, 19 Oct 2020 12:12:19 +0200 Subject: [PATCH] rewrite sentry to support info messages --- guard.go | 3 +- helpers.go | 38 +++++++++++++ middlewares.go | 110 +++++--------------------------------- sentry.go | 141 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 98 deletions(-) create mode 100644 sentry.go diff --git a/guard.go b/guard.go index 57389ed..bf08806 100644 --- a/guard.go +++ b/guard.go @@ -31,6 +31,7 @@ type ( Config *Config Status *CmdStatus + Reporter *Reporter } // CmdStatus is the commands status @@ -69,7 +70,7 @@ func main() { r := chained( runner, timeout, validateStdout, validateStderr, quietIgnore, - sentryHandler, lockfile, headerize, combineLogs, insertUUID, + lockfile, sentryHandler, headerize, combineLogs, insertUUID, writeSyslog, setupLogs, ) err := r(context.Background(), &cr) diff --git a/helpers.go b/helpers.go index c9c7bf9..0820f2c 100644 --- a/helpers.go +++ b/helpers.go @@ -5,7 +5,11 @@ import ( "errors" "fmt" "io" + "io/ioutil" + "os" + "strconv" "strings" + "syscall" "time" "github.com/robfig/cron" @@ -79,3 +83,37 @@ func isQuiet(cr *CmdRequest) (bool, error) { } return false, nil } + +// handleLockfile validates the lockfile and checks if the command should be run +func handleExistingLockfile(cr *CmdRequest) (bool, error) { + _, statErr := os.Stat(cr.Lockfile) + if statErr == nil { + pidBytes, err := ioutil.ReadFile(cr.Lockfile) + if err != nil { + return false, fmt.Errorf("unable to read lockfile: %s", err) + } + pid, err := strconv.Atoi(string(pidBytes)) + if err != nil { + return false, fmt.Errorf("unable to read pidfile: %s", err) + } + proc, err := os.FindProcess(pid) + if err != nil { + return false, fmt.Errorf("process(%d) from pidfile missing: %s", pid, err) + } + err = proc.Signal(syscall.Signal(0)) + if err == nil { + _, _ = fmt.Fprintf(cr.Status.Combined, "cron is still running, pid: %d", pid) + return false, nil + } else { + // if we have an orphaned pid, we try to report that to our reporter and continue + logErr := fmt.Errorf("process(%d) from pidfile missing: %s", pid, err) + if cr.Reporter != nil { + cr.Reporter.Info(logErr) + } + return true, nil + } + } else if !os.IsNotExist(statErr) { + return false, fmt.Errorf("unable to handle lockfile: %s", statErr) + } + return true, nil +} diff --git a/middlewares.go b/middlewares.go index 19e99d0..647bdde 100644 --- a/middlewares.go +++ b/middlewares.go @@ -4,21 +4,14 @@ import ( "bufio" "bytes" "context" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "io" - "io/ioutil" "log" "log/syslog" "os" - "strconv" - "strings" - "syscall" "time" - "github.com/getsentry/sentry-go" "golang.org/x/sync/errgroup" ) @@ -70,7 +63,7 @@ func insertUUID(g GuardFunc) GuardFunc { if cr.ErrFileHideUUID { return g(ctx, cr) } - combined := newUUIDPrefixer(cr.Status.Combined) + combined := newUUIDPrefixer(cr.Status.Combined) cr.Status.Combined = combined return g(ctx, cr) } @@ -117,31 +110,16 @@ func headerize(g GuardFunc) GuardFunc { func lockfile(g GuardFunc) GuardFunc { return func(ctx context.Context, cr *CmdRequest) (err error) { if cr.Lockfile != "" { - _, statErr := os.Stat(cr.Lockfile) - if statErr == nil { - pidBytes, err := ioutil.ReadFile(cr.Lockfile) - if err != nil { - return fmt.Errorf("unable to read lockfile: %s", err) - } - pid, err := strconv.Atoi(string(pidBytes)) - if err != nil { - return fmt.Errorf("unable to read pidfile: %s", err) - } - proc, err := os.FindProcess(pid) - if err != nil { - return fmt.Errorf("process(%d) from pidfile missing: %s", pid, err) - } - err = proc.Signal(syscall.Signal(0)) - if err != nil { - return fmt.Errorf("process(%d) from pidfile missing: %s", pid, err) - } - _, _ = fmt.Fprintf(cr.Status.Combined, "cron is still running, pid: %d", pid) + run, err := handleExistingLockfile(cr) + if err != nil { + return err + } + if !run { return nil - } else if !os.IsNotExist(statErr) { - return fmt.Errorf("unable to handle lockfile: %s", statErr) } + pid := os.Getpid() - lockfile, err := os.OpenFile(cr.Lockfile, os.O_CREATE|os.O_RDWR, 0600) + lockfile, err := os.OpenFile(cr.Lockfile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) if err != nil { return fmt.Errorf("unable to open lockfile: %s", err) } @@ -157,80 +135,18 @@ func lockfile(g GuardFunc) GuardFunc { } } +// sentryHandler redirects all errors to a sentry if configured func sentryHandler(g GuardFunc) GuardFunc { return func(ctx context.Context, cr *CmdRequest) (err error) { - // check if envar is set - sentryDSN, ok := os.LookupEnv("CRONGUARD_SENTRY_DSN") - if !ok && cr.Config != nil { - sentryDSN = cr.Config.SentryDSN - } - if sentryDSN == "" { + reporter, reporterErr := newReporter(cr) + if reporterErr != nil { return g(ctx, cr) } - // wrap buffers - start := time.Now() - combined := bytes.NewBuffer([]byte{}) - stderr := bytes.NewBuffer([]byte{}) - cr.Status.Stderr = io.MultiWriter(stderr, combined, cr.Status.Stderr) - cr.Status.Stdout = io.MultiWriter(combined, cr.Status.Stdout) - - // prepare sentry - sentryErr := sentry.Init(sentry.ClientOptions{ - Dsn: sentryDSN, - Transport: sentry.NewHTTPSyncTransport(), - }) - if sentryErr != nil { - fmt.Fprintf(cr.Status.Stderr, "cronguard: unable to connect to sentry: %s\n", sentryErr) - fmt.Fprintf(cr.Status.Stderr, "cronguard: running cron anyways\n") - } - + cr.Reporter = reporter err = g(ctx, cr) - - // try to log to sentry - if err != nil && sentryErr == nil { - // gather data - hostname, _ := os.Hostname() - if hostname == "" { - hostname = "no-hostname" - } - hostname = strings.SplitN(hostname, ".", 2)[0] - cmd := cr.Command - if len(cmd) > 32 { - cmd = fmt.Sprintf("%s%s", cmd[0:30], "...") - } - cmdHash := sha256.New() - cmdHash.Write([]byte(cr.Command)) - cmdHash.Write([]byte(hostname)) - hash := hex.EncodeToString(cmdHash.Sum(nil)) - - // add data to message - sentry.ConfigureScope(func(scope *sentry.Scope) { - scope.SetExtra("time_start", start) - scope.SetExtra("time_end", time.Now()) - scope.SetExtra("time_duration", time.Since(start).String()) - scope.SetExtra("out_combined", combined.String()) - scope.SetExtra("out_stderr", stderr.String()) - scope.SetExtra("command", cr.Command) - scope.SetFingerprint([]string{hash}) - }) - name := fmt.Sprintf( - "%s: %s (%s)", - hostname, - cmd, - err.Error(), - ) - _ = sentry.CaptureMessage(name) - - // hide error if messages are successfully flushed to sentry - flushed := sentry.Flush(30 * time.Second) - if flushed { - return nil - } - } - return err + return reporter.Finish(err) } - } // quietIgnore allows to ignore errors on lower settings if flag is set diff --git a/sentry.go b/sentry.go new file mode 100644 index 0000000..ae617a0 --- /dev/null +++ b/sentry.go @@ -0,0 +1,141 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "hash" + "io" + "os" + "strings" + "time" + + "github.com/getsentry/sentry-go" +) + +type ( + Reporter struct { + sentryDSN string + start time.Time + hostname string + cmd string + hash hash.Hash + + combined *bytes.Buffer + stderr *bytes.Buffer + } +) + +// newReporter creates a new Sentry client +func newReporter(cr *CmdRequest) (*Reporter, error) { + sentryDSN, ok := os.LookupEnv("CRONGUARD_SENTRY_DSN") + if !ok && cr.Config != nil { + sentryDSN = cr.Config.SentryDSN + } + if sentryDSN == "" { + return nil, fmt.Errorf("no config provided") + } + + // data + hostname, _ := os.Hostname() + if hostname == "" { + hostname = "no-hostname" + } + hostname = strings.SplitN(hostname, ".", 2)[0] + hash := sha256.New() + hash.Write([]byte(cr.Command)) + hash.Write([]byte(hostname)) + cmd := cr.Command + if len(cmd) > 32 { + cmd = fmt.Sprintf("%s%s", cmd[0:30], "...") + } + + // setup sentry + sentryErr := sentry.Init(sentry.ClientOptions{ + Dsn: sentryDSN, + Transport: sentry.NewHTTPSyncTransport(), + }) + if sentryErr != nil { + fmt.Fprintf(cr.Status.Stderr, "cronguard: unable to connect to sentry: %s\n", sentryErr) + fmt.Fprintf(cr.Status.Stderr, "cronguard: running cron anyways\n") + return nil, fmt.Errorf("unable to connect to sentry") + } + + // wrap buffers + start := time.Now() + combined := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + cr.Status.Stderr = io.MultiWriter(stderr, combined, cr.Status.Stderr) + cr.Status.Stdout = io.MultiWriter(combined, cr.Status.Stdout) + + // set known sentry extras + sentry.ConfigureScope(func(scope *sentry.Scope) { + scope.SetExtra("time_start", start) + scope.SetExtra("command", cr.Command) + }) + + return &Reporter{ + sentryDSN: sentryDSN, + start: start, + hostname: hostname, + cmd: cmd, + hash: hash, + combined: combined, + stderr: stderr, + }, nil +} + +// Finish reports the final status to sentry if err != nil +func (r *Reporter) Finish(err error) error { + if err == nil { + return nil + } + return r.report(err, finishLevel) +} + +// Info reports a Info status to sentry +func (r *Reporter) Info(err error) error { + return r.report(err, infoLevel) +} + +// reportLevel is used by reporter to disingques +type reportLevel = string + +const ( + // infoLevel is an information that will be send to sentry + infoLevel reportLevel = "info" + // finishLevel is used to tell the reporter that the cron has finished + finishLevel = "finish" +) + +// report reports any error message to sentry +func (r *Reporter) report(err error, level reportLevel) error { + // prepare sentry information + name := "" + extra := map[string]interface{}{} + if level == finishLevel { + name = fmt.Sprintf("%s: %s (%s)", r.hostname, r.cmd, err.Error()) + extra["time_end"] = time.Now() + extra["time_duration"] = time.Since(r.start).String() + extra["out_combined"] = r.combined.String() + extra["out_stderr"] = r.stderr.String() + } else { + name = fmt.Sprintf("%s (%s): %s (%s)", r.hostname, level, r.cmd, err.Error()) + } + + // sentry + hash := hex.EncodeToString(r.hash.Sum([]byte(level))) + sentry.ConfigureScope(func(scope *sentry.Scope) { + scope.SetFingerprint([]string{hash}) + scope.SetExtras(extra) + }) + _ = sentry.CaptureMessage(name) + + // hide error if messages are successfully flushed to sentry + flushed := sentry.Flush(30 * time.Second) + if !flushed { + return err + } + return nil +}