From 4a3a4145a46595fda1c685beaa822933bf2cc653 Mon Sep 17 00:00:00 2001 From: smantic Date: Fri, 1 Nov 2024 21:32:15 -0500 Subject: [PATCH] add messages used histogram --- command/talk.go | 3 ++- main.go | 17 +++++++------- metrics/metrics.go | 21 +++++++++-------- privmsg.go | 4 ++-- robot.go | 57 +++++++++++++++++++++++++++++++++++++++++++++- tmi.go | 6 ++--- 6 files changed, 83 insertions(+), 25 deletions(-) diff --git a/command/talk.go b/command/talk.go index 7c5bb9e..dc53fbc 100644 --- a/command/talk.go +++ b/command/talk.go @@ -61,8 +61,9 @@ func speakCmd(ctx context.Context, robo *Robot, call *Invocation, effect string) } // block the generated message from being later recognized as a meme. call.Channel.Memery.Block(call.Message.Time(), s) + robo.Metrics.SpeakLatency.Observe(time.Since(start).Seconds(), call.Channel.Send, fmt.Sprintf("%t", len(call.Args["prompt"]) == 0)) + robo.Metrics.UsedMessagesForGeneration.Observe(float64(len(trace))) robo.Log.InfoContext(ctx, "speak", "in", call.Channel.Name, "text", m, "emote", e) - robo.Metrics.SpeakLatency.Observe(time.Since(start).Seconds(), call.Channel.Name) return m + " " + e } diff --git a/main.go b/main.go index 58b56ce..5c6d997 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/urfave/cli/v3" "golang.org/x/sync/errgroup" "zombiezen.com/go/sqlite/sqlitex" @@ -323,7 +322,7 @@ func loggerFromFlags(cmd *cli.Command) *slog.Logger { func newMetrics() *metrics.Metrics { return &metrics.Metrics{ TMIMsgsCount: metrics.NewPromCounter( - promauto.NewCounter(prometheus.CounterOpts{ + prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "robot", Subsystem: "tmi", Name: "messages", @@ -331,7 +330,7 @@ func newMetrics() *metrics.Metrics { }), ), TMICommandCount: metrics.NewPromCounter( - promauto.NewCounter(prometheus.CounterOpts{ + prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "robot", Subsystem: "tmi", Name: "commands", @@ -339,7 +338,7 @@ func newMetrics() *metrics.Metrics { }), ), LearnedCount: metrics.NewPromCounter( - promauto.NewCounter(prometheus.CounterOpts{ + prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "robot", Subsystem: "brain", Name: "learned", @@ -347,7 +346,7 @@ func newMetrics() *metrics.Metrics { }), ), ForgotCount: metrics.NewPromCounter( - promauto.NewCounter(prometheus.CounterOpts{ + prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "robot", Subsystem: "brain", Name: "forgot", @@ -355,18 +354,18 @@ func newMetrics() *metrics.Metrics { }), ), SpeakLatency: metrics.NewPromObserverVec( - promauto.NewHistogramVec(prometheus.HistogramOpts{ + prometheus.NewHistogramVec(prometheus.HistogramOpts{ Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.5, 1, 5, 10}, Namespace: "robot", Subsystem: "commands", Name: "speak-latency", Help: "How long it takes for robot to speak once prompted in seconds", - }, []string{"channel"}, + }, []string{"channel", "empty-prompt"}, ), ), LearnLatency: metrics.NewPromObserverVec( - promauto.NewHistogramVec(prometheus.HistogramOpts{ - Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.5, 1, 5, 10}, + prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Buckets: []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, Namespace: "robot", Subsystem: "brain", Name: "learn-latency", diff --git a/metrics/metrics.go b/metrics/metrics.go index 94fd767..5e75d39 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,12 +11,13 @@ type Observer interface { } type Metrics struct { - TMIMsgsCount Observer - TMICommandCount Observer - LearnedCount Observer - ForgotCount Observer - SpeakLatency Observer - LearnLatency Observer + TMIMsgsCount Observer + TMICommandCount Observer + LearnedCount Observer + ForgotCount Observer + SpeakLatency Observer + LearnLatency Observer + UsedMessagesForGeneration Observer } func (m Metrics) Collectors() []prometheus.Collector { @@ -31,12 +32,14 @@ func (m Metrics) Collectors() []prometheus.Collector { } // for testing purposes. -type StubMetric struct{} +type StubMetric struct { + Name string +} -func (s *StubMetric) Observe(val float64, tags ...string) { return } +func (s *StubMetric) Observe(val float64, tags ...string) {} func (s *StubMetric) Describe(c chan<- *prometheus.Desc) { - c <- prometheus.NewDesc("stub metric", "", nil, nil) + c <- prometheus.NewDesc(s.Name, "", nil, nil) } func (s *StubMetric) Collect(c chan<- prometheus.Metric) { diff --git a/privmsg.go b/privmsg.go index e8a6ae2..f917fd1 100644 --- a/privmsg.go +++ b/privmsg.go @@ -256,14 +256,14 @@ func (robo *Robot) learn(ctx context.Context, log *slog.Logger, ch *channel.Chan log.DebugContext(ctx, "no learn tag") return } - start := time.Now() user := hasher.Hash(new(userhash.Hash), msg.Sender, msg.To, msg.Time()) + start := time.Now() if err := brain.Learn(ctx, robo.brain, ch.Learn, msg.ID, *user, msg.Time(), brain.Tokens(nil, msg.Text)); err != nil { log.ErrorContext(ctx, "failed to learn", slog.Any("err", err)) return } + robo.Metrics.LearnLatency.Observe(time.Since(start).Seconds(), ch.Learn) robo.Metrics.LearnedCount.Observe(1) - robo.Metrics.LearnLatency.Observe(time.Since(start).Seconds(), ch.Name) } // sendTMI sends a message to TMI after waiting for the global rate limit. diff --git a/robot.go b/robot.go index 60c2dcf..85b0765 100644 --- a/robot.go +++ b/robot.go @@ -13,6 +13,8 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/time/rate" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/zephyrtronium/robot/auth" "github.com/zephyrtronium/robot/brain" "github.com/zephyrtronium/robot/channel" @@ -78,7 +80,60 @@ func New(usersKey []byte, poolSize int) *Robot { channels: syncmap.New[string, *channel.Channel](), works: make(chan chan func(context.Context), poolSize), hashes: func() userhash.Hasher { return userhash.New(usersKey) }, - Metrics: newMetrics(), + Metrics: &metrics.Metrics{ + TMIMsgsCount: metrics.NewPromCounter( + promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "robot", + Subsystem: "tmi", + Name: "messages", + Help: "Number of PRIVMSGs received from TMI.", + }), + ), + TMICommandCount: metrics.NewPromCounter( + promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "robot", + Subsystem: "tmi", + Name: "commands", + Help: "Number of command invocations received in Twitch chat.", + }), + ), + LearnedCount: metrics.NewPromCounter( + promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "robot", + Subsystem: "brain", + Name: "learned", + Help: "Number of messages learned.", + }), + ), + ForgotCount: metrics.NewPromCounter( + promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "robot", + Subsystem: "brain", + Name: "forgot", + Help: "Number of individual messages deleted. Does not include messages deleted by user or time.", + }), + ), + SpeakLatency: metrics.NewPromObserverVec( + promauto.NewHistogramVec(prometheus.HistogramOpts{ + Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.5, 1, 5, 10}, + Namespace: "robot", + Subsystem: "commands", + Name: "speak-latency", + Help: "How long it takes for robot to speak once prompted in seconds", + }, []string{"send-tag"}, + ), + ), + LearnLatency: metrics.NewPromObserverVec( + promauto.NewHistogramVec(prometheus.HistogramOpts{ + Buckets: []float64{0.01, 0.05, 0.1, 0.2, 0.5, 1, 5, 10}, + Namespace: "robot", + Subsystem: "brain", + Name: "learn-latency", + Help: "How long it takes robot to learn a non discarded message in seconds", + }, []string{"channel"}, + ), + ), + }, } } diff --git a/tmi.go b/tmi.go index 71964e5..f38d575 100644 --- a/tmi.go +++ b/tmi.go @@ -158,7 +158,7 @@ func (robo *Robot) clearmsg(ctx context.Context, group *errgroup.Group, msg *tmi if u != robo.tmi.name { // Forget a message from someone else. log.InfoContext(ctx, "forget message", slog.String("tag", ch.Learn), slog.String("id", t)) - forget(ctx, log, robo.brain, ch.Learn, robo.Metrics.ForgotCount, t) + forget(ctx, log, robo.Metrics.ForgotCount, robo.brain, ch.Learn, t) return } // Forget a message from the robo. @@ -178,12 +178,12 @@ func (robo *Robot) clearmsg(ctx context.Context, group *errgroup.Group, msg *tmi return } log.InfoContext(ctx, "forget trace", slog.String("tag", ch.Send), slog.Any("spoken", tm), slog.Any("trace", trace)) - forget(ctx, log, robo.brain, ch.Send, robo.Metrics.ForgotCount, trace...) + forget(ctx, log, robo.Metrics.ForgotCount, robo.brain, ch.Send, trace...) } robo.enqueue(ctx, group, work) } -func forget(ctx context.Context, log *slog.Logger, brain brain.Brain, tag string, forgetCount metrics.Observer, trace ...string) { +func forget(ctx context.Context, log *slog.Logger, forgetCount metrics.Observer, brain brain.Brain, tag string, trace ...string) { forgetCount.Observe(1) for _, id := range trace { err := brain.ForgetMessage(ctx, tag, id)