Skip to content

Commit

Permalink
rewrite sentry to support info messages
Browse files Browse the repository at this point in the history
  • Loading branch information
foosinn committed Oct 19, 2020
1 parent e319cc9 commit 88cbd30
Show file tree
Hide file tree
Showing 4 changed files with 194 additions and 98 deletions.
3 changes: 2 additions & 1 deletion guard.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type (
Config *Config

Status *CmdStatus
Reporter *Reporter
}

// CmdStatus is the commands status
Expand Down Expand Up @@ -69,7 +70,7 @@ func main() {

r := chained(
runner, timeout, validateStdout, validateStderr, quietIgnore,
sentryHandler, lockfile, headerize, combineLogs, insertUUID,
lockfile, sentryHandler, headerize, combineLogs, insertUUID,
writeSyslog, setupLogs,
)
err := r(context.Background(), &cr)
Expand Down
38 changes: 38 additions & 0 deletions helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"strconv"
"strings"
"syscall"
"time"

"github.com/robfig/cron"
Expand Down Expand Up @@ -79,3 +83,37 @@ func isQuiet(cr *CmdRequest) (bool, error) {
}
return false, nil
}

// handleLockfile validates the lockfile and checks if the command should be run
func handleExistingLockfile(cr *CmdRequest) (bool, error) {
_, statErr := os.Stat(cr.Lockfile)
if statErr == nil {
pidBytes, err := ioutil.ReadFile(cr.Lockfile)
if err != nil {
return false, fmt.Errorf("unable to read lockfile: %s", err)
}
pid, err := strconv.Atoi(string(pidBytes))
if err != nil {
return false, fmt.Errorf("unable to read pidfile: %s", err)
}
proc, err := os.FindProcess(pid)
if err != nil {
return false, fmt.Errorf("process(%d) from pidfile missing: %s", pid, err)
}
err = proc.Signal(syscall.Signal(0))
if err == nil {
_, _ = fmt.Fprintf(cr.Status.Combined, "cron is still running, pid: %d", pid)
return false, nil
} else {
// if we have an orphaned pid, we try to report that to our reporter and continue
logErr := fmt.Errorf("process(%d) from pidfile missing: %s", pid, err)
if cr.Reporter != nil {
cr.Reporter.Info(logErr)
}
return true, nil
}
} else if !os.IsNotExist(statErr) {
return false, fmt.Errorf("unable to handle lockfile: %s", statErr)
}
return true, nil
}
110 changes: 13 additions & 97 deletions middlewares.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,14 @@ import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"log/syslog"
"os"
"strconv"
"strings"
"syscall"
"time"

"github.com/getsentry/sentry-go"
"golang.org/x/sync/errgroup"
)

Expand Down Expand Up @@ -70,7 +63,7 @@ func insertUUID(g GuardFunc) GuardFunc {
if cr.ErrFileHideUUID {
return g(ctx, cr)
}
combined := newUUIDPrefixer(cr.Status.Combined)
combined := newUUIDPrefixer(cr.Status.Combined)
cr.Status.Combined = combined
return g(ctx, cr)
}
Expand Down Expand Up @@ -117,31 +110,16 @@ func headerize(g GuardFunc) GuardFunc {
func lockfile(g GuardFunc) GuardFunc {
return func(ctx context.Context, cr *CmdRequest) (err error) {
if cr.Lockfile != "" {
_, statErr := os.Stat(cr.Lockfile)
if statErr == nil {
pidBytes, err := ioutil.ReadFile(cr.Lockfile)
if err != nil {
return fmt.Errorf("unable to read lockfile: %s", err)
}
pid, err := strconv.Atoi(string(pidBytes))
if err != nil {
return fmt.Errorf("unable to read pidfile: %s", err)
}
proc, err := os.FindProcess(pid)
if err != nil {
return fmt.Errorf("process(%d) from pidfile missing: %s", pid, err)
}
err = proc.Signal(syscall.Signal(0))
if err != nil {
return fmt.Errorf("process(%d) from pidfile missing: %s", pid, err)
}
_, _ = fmt.Fprintf(cr.Status.Combined, "cron is still running, pid: %d", pid)
run, err := handleExistingLockfile(cr)
if err != nil {
return err
}
if !run {
return nil
} else if !os.IsNotExist(statErr) {
return fmt.Errorf("unable to handle lockfile: %s", statErr)
}

pid := os.Getpid()
lockfile, err := os.OpenFile(cr.Lockfile, os.O_CREATE|os.O_RDWR, 0600)
lockfile, err := os.OpenFile(cr.Lockfile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
if err != nil {
return fmt.Errorf("unable to open lockfile: %s", err)
}
Expand All @@ -157,80 +135,18 @@ func lockfile(g GuardFunc) GuardFunc {
}
}

// sentryHandler redirects all errors to a sentry if configured
func sentryHandler(g GuardFunc) GuardFunc {
return func(ctx context.Context, cr *CmdRequest) (err error) {
// check if envar is set
sentryDSN, ok := os.LookupEnv("CRONGUARD_SENTRY_DSN")
if !ok && cr.Config != nil {
sentryDSN = cr.Config.SentryDSN
}
if sentryDSN == "" {
reporter, reporterErr := newReporter(cr)
if reporterErr != nil {
return g(ctx, cr)
}

// wrap buffers
start := time.Now()
combined := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cr.Status.Stderr = io.MultiWriter(stderr, combined, cr.Status.Stderr)
cr.Status.Stdout = io.MultiWriter(combined, cr.Status.Stdout)

// prepare sentry
sentryErr := sentry.Init(sentry.ClientOptions{
Dsn: sentryDSN,
Transport: sentry.NewHTTPSyncTransport(),
})
if sentryErr != nil {
fmt.Fprintf(cr.Status.Stderr, "cronguard: unable to connect to sentry: %s\n", sentryErr)
fmt.Fprintf(cr.Status.Stderr, "cronguard: running cron anyways\n")
}

cr.Reporter = reporter
err = g(ctx, cr)

// try to log to sentry
if err != nil && sentryErr == nil {
// gather data
hostname, _ := os.Hostname()
if hostname == "" {
hostname = "no-hostname"
}
hostname = strings.SplitN(hostname, ".", 2)[0]
cmd := cr.Command
if len(cmd) > 32 {
cmd = fmt.Sprintf("%s%s", cmd[0:30], "...")
}
cmdHash := sha256.New()
cmdHash.Write([]byte(cr.Command))
cmdHash.Write([]byte(hostname))
hash := hex.EncodeToString(cmdHash.Sum(nil))

// add data to message
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetExtra("time_start", start)
scope.SetExtra("time_end", time.Now())
scope.SetExtra("time_duration", time.Since(start).String())
scope.SetExtra("out_combined", combined.String())
scope.SetExtra("out_stderr", stderr.String())
scope.SetExtra("command", cr.Command)
scope.SetFingerprint([]string{hash})
})
name := fmt.Sprintf(
"%s: %s (%s)",
hostname,
cmd,
err.Error(),
)
_ = sentry.CaptureMessage(name)

// hide error if messages are successfully flushed to sentry
flushed := sentry.Flush(30 * time.Second)
if flushed {
return nil
}
}
return err
return reporter.Finish(err)
}

}

// quietIgnore allows to ignore errors on lower settings if flag is set
Expand Down
141 changes: 141 additions & 0 deletions sentry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package main

import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"io"
"os"
"strings"
"time"

"github.com/getsentry/sentry-go"
)

type (
Reporter struct {
sentryDSN string
start time.Time
hostname string
cmd string
hash hash.Hash

combined *bytes.Buffer
stderr *bytes.Buffer
}
)

// newReporter creates a new Sentry client
func newReporter(cr *CmdRequest) (*Reporter, error) {
sentryDSN, ok := os.LookupEnv("CRONGUARD_SENTRY_DSN")
if !ok && cr.Config != nil {
sentryDSN = cr.Config.SentryDSN
}
if sentryDSN == "" {
return nil, fmt.Errorf("no config provided")
}

// data
hostname, _ := os.Hostname()
if hostname == "" {
hostname = "no-hostname"
}
hostname = strings.SplitN(hostname, ".", 2)[0]
hash := sha256.New()
hash.Write([]byte(cr.Command))
hash.Write([]byte(hostname))
cmd := cr.Command
if len(cmd) > 32 {
cmd = fmt.Sprintf("%s%s", cmd[0:30], "...")
}

// setup sentry
sentryErr := sentry.Init(sentry.ClientOptions{
Dsn: sentryDSN,
Transport: sentry.NewHTTPSyncTransport(),
})
if sentryErr != nil {
fmt.Fprintf(cr.Status.Stderr, "cronguard: unable to connect to sentry: %s\n", sentryErr)
fmt.Fprintf(cr.Status.Stderr, "cronguard: running cron anyways\n")
return nil, fmt.Errorf("unable to connect to sentry")
}

// wrap buffers
start := time.Now()
combined := bytes.NewBuffer([]byte{})
stderr := bytes.NewBuffer([]byte{})
cr.Status.Stderr = io.MultiWriter(stderr, combined, cr.Status.Stderr)
cr.Status.Stdout = io.MultiWriter(combined, cr.Status.Stdout)

// set known sentry extras
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetExtra("time_start", start)
scope.SetExtra("command", cr.Command)
})

return &Reporter{
sentryDSN: sentryDSN,
start: start,
hostname: hostname,
cmd: cmd,
hash: hash,
combined: combined,
stderr: stderr,
}, nil
}

// Finish reports the final status to sentry if err != nil
func (r *Reporter) Finish(err error) error {
if err == nil {
return nil
}
return r.report(err, finishLevel)
}

// Info reports a Info status to sentry
func (r *Reporter) Info(err error) error {
return r.report(err, infoLevel)
}

// reportLevel is used by reporter to disingques
type reportLevel = string

const (
// infoLevel is an information that will be send to sentry
infoLevel reportLevel = "info"
// finishLevel is used to tell the reporter that the cron has finished
finishLevel = "finish"
)

// report reports any error message to sentry
func (r *Reporter) report(err error, level reportLevel) error {
// prepare sentry information
name := ""
extra := map[string]interface{}{}
if level == finishLevel {
name = fmt.Sprintf("%s: %s (%s)", r.hostname, r.cmd, err.Error())
extra["time_end"] = time.Now()
extra["time_duration"] = time.Since(r.start).String()
extra["out_combined"] = r.combined.String()
extra["out_stderr"] = r.stderr.String()
} else {
name = fmt.Sprintf("%s (%s): %s (%s)", r.hostname, level, r.cmd, err.Error())
}

// sentry
hash := hex.EncodeToString(r.hash.Sum([]byte(level)))
sentry.ConfigureScope(func(scope *sentry.Scope) {
scope.SetFingerprint([]string{hash})
scope.SetExtras(extra)
})
_ = sentry.CaptureMessage(name)

// hide error if messages are successfully flushed to sentry
flushed := sentry.Flush(30 * time.Second)
if !flushed {
return err
}
return nil
}

0 comments on commit 88cbd30

Please sign in to comment.