Skip to content

Commit

Permalink
Add alert sending timeout and dingtalk robot (#40)
Browse files Browse the repository at this point in the history
  • Loading branch information
wanliqun authored May 22, 2024
1 parent 2eb5218 commit 6ed9a9b
Show file tree
Hide file tree
Showing 11 changed files with 201 additions and 19 deletions.
4 changes: 3 additions & 1 deletion alert/alert.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package alert

import (
"context"

viperutil "github.com/Conflux-Chain/go-conflux-util/viper"
"github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -42,7 +44,7 @@ const (
type Channel interface {
Name() string
Type() ChannelType
Send(note *Notification) error
Send(context.Context, *Notification) error
}

// Notification represents core information for an alert.
Expand Down
17 changes: 11 additions & 6 deletions alert/dingtalk.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package alert

import (
"context"

"github.com/Conflux-Chain/go-conflux-util/alert/dingtalk"
"github.com/pkg/errors"
"github.com/royeo/dingrobot"
)

var (
Expand All @@ -21,10 +23,15 @@ type DingTalkChannel struct {
Formatter Formatter // message formatter
ID string // channel id
Config DingTalkConfig // channel config

bot *dingtalk.Robot
}

func NewDingTalkChannel(chID string, fmt Formatter, conf DingTalkConfig) *DingTalkChannel {
return &DingTalkChannel{ID: chID, Formatter: fmt, Config: conf}
return &DingTalkChannel{
ID: chID, Formatter: fmt, Config: conf,
bot: dingtalk.NewRobot(conf.Webhook, conf.Secret),
}
}

func (dtc *DingTalkChannel) Name() string {
Expand All @@ -35,13 +42,11 @@ func (dtc *DingTalkChannel) Type() ChannelType {
return ChannelTypeDingTalk
}

func (dtc *DingTalkChannel) Send(note *Notification) error {
func (dtc *DingTalkChannel) Send(ctx context.Context, note *Notification) error {
msg, err := dtc.Formatter.Format(note)
if err != nil {
return errors.WithMessage(err, "failed to format alert msg")
}

dingRobot := dingrobot.NewRobot(dtc.Config.Webhook)
dingRobot.SetSecret(dtc.Config.Secret)
return dingRobot.SendMarkdown(note.Title, msg, dtc.Config.AtMobiles, dtc.Config.IsAtAll)
return dtc.bot.SendMarkdown(ctx, note.Title, msg, dtc.Config.AtMobiles, dtc.Config.IsAtAll)
}
102 changes: 102 additions & 0 deletions alert/dingtalk/bot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package dingtalk

import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"time"

"github.com/pkg/errors"
)

// Robot represents a dingtalk custom robot that can send messages to groups.
type Robot struct {
webHook string
secret string
}

func NewRobot(webhook, secrect string) *Robot {
return &Robot{webHook: webhook, secret: secrect}
}

// SendMarkdown send a markdown type message.
func (r Robot) SendMarkdown(ctx context.Context, title, text string, atMobiles []string, isAtAll bool) error {
return r.send(ctx, &markdownMessage{
MsgType: msgTypeMarkdown,
Markdown: markdownParams{
Title: title,
Text: text,
},
At: atParams{
AtMobiles: atMobiles,
IsAtAll: isAtAll,
},
})
}

type dingResponse struct {
Errcode int `json:"errcode"`
Errmsg string `json:"errmsg"`
}

func (r Robot) send(ctx context.Context, msg interface{}) error {
jm, err := json.Marshal(msg)
if err != nil {
return errors.WithMessage(err, "failed to marshal message")
}

webURL := r.webHook
if len(r.secret) != 0 {
webURL += genSignedURL(r.secret)
}

req, errRequest := http.NewRequestWithContext(ctx, http.MethodPost, webURL, bytes.NewReader(jm))
if errRequest != nil {
return errors.WithMessage(errRequest, "failed to create request")
}

req.Header.Add("Content-Type", "application/json")
resp, errDo := http.DefaultClient.Do(req)
if errDo != nil {
return errors.WithMessage(errDo, "failed to do http request")
}
defer resp.Body.Close()

body, errReadBody := io.ReadAll(resp.Body)
if errReadBody != nil {
return errors.WithMessage(errReadBody, "failed to read http response body")
}

var dr dingResponse
err = json.Unmarshal(body, &dr)
if err != nil {
return err
}
if dr.Errcode != 0 {
return fmt.Errorf("dingrobot send failed: %v", dr.Errmsg)
}

return nil
}

func genSignedURL(secret string) string {
timeStr := fmt.Sprintf("%d", time.Now().UnixNano()/1e6)
sign := fmt.Sprintf("%s\n%s", timeStr, secret)
signData := computeHmacSha256(sign, secret)
encodeURL := url.QueryEscape(signData)
return fmt.Sprintf("&timestamp=%s&sign=%s", timeStr, encodeURL)
}

func computeHmacSha256(message string, secret string) string {
key := []byte(secret)
h := hmac.New(sha256.New, key)
h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
43 changes: 43 additions & 0 deletions alert/dingtalk/bot_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package dingtalk

import (
"context"
"os"
"testing"
"time"

"github.com/stretchr/testify/assert"
)

var (
robot *Robot
)

// Please set the following enviroments before running tests:
// `TEST_DINGTALK_WEBHOOK`: DingTalk webhook;
// `TEST_DINGTALK_SECRET`: DingTalk secret.

func TestMain(m *testing.M) {
webhook := os.Getenv("TEST_DINGTALK_WEBHOOK")
secrect := os.Getenv("TEST_DINGTALK_SECRET")

if len(webhook) > 0 && len(secrect) > 0 {
robot = NewRobot(webhook, secrect)
}

os.Exit(m.Run())
}

func TestSendMarkdown(t *testing.T) {
if robot == nil {
t.SkipNow()
return
}

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()

// Please manually check if message sent to dingtalk group chat
err := robot.SendMarkdown(ctx, "test", "# Hello, test!", nil, false)
assert.NoError(t, err)
}
21 changes: 21 additions & 0 deletions alert/dingtalk/message.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package dingtalk

const (
msgTypeMarkdown = "markdown"
)

type atParams struct {
AtMobiles []string `json:"atMobiles,omitempty"`
IsAtAll bool `json:"isAtAll,omitempty"`
}

type markdownMessage struct {
MsgType string `json:"msgtype"`
Markdown markdownParams `json:"markdown"`
At atParams `json:"at"`
}

type markdownParams struct {
Title string `json:"title"`
Text string `json:"text"`
}
4 changes: 2 additions & 2 deletions alert/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ func (tc *TelegramChannel) Type() ChannelType {
return ChannelTypeTelegram
}

func (tc *TelegramChannel) Send(note *Notification) error {
func (tc *TelegramChannel) Send(ctx context.Context, note *Notification) error {
msg, err := tc.Formatter.Format(note)
if err != nil {
return errors.WithMessage(err, "failed to format alert msg")
}

_, err = tc.bot.SendMessage(context.Background(), &bot.SendMessageParams{
_, err = tc.bot.SendMessage(ctx, &bot.SendMessageParams{
ChatID: tc.Config.ChatId,
Text: msg,
ParseMode: models.ParseModeMarkdown,
Expand Down
7 changes: 5 additions & 2 deletions config/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,21 @@
# level: warn
# # Default notification channels
# channels: []
# # Maximum request timeout allowed to send alert.
# SendTimeout: 3s

# # Async worker options for sending alert
# async:
# # The number of worker goroutines (set `0` to turn off async mode).
# numWorkers: 1
# # The maximum number of queued jobs.
# queueSize: 60
# # Maximum Timeout allowed to gracefully stop..
# # Maximum timeout allowed to gracefully stop.
# StopTimeout: 5s

# Alert Configurations
# alert:
# # Custom tags are usually used to differentiate between different networks and enviroments
# # Custom tags are usually used to differentiate between different networks and environments
# # such as mainnet/testnet, prod/test/dev or any custom info for more details.
# customTags: [dev]
# # Channels: Notification channels
Expand Down
1 change: 0 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ require (
github.com/mcuadros/go-defaults v1.2.0
github.com/mitchellh/mapstructure v1.4.3
github.com/pkg/errors v0.9.1
github.com/royeo/dingrobot v1.0.1-0.20191230075228-c90a788ca8fd
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.10.0
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -399,8 +399,6 @@ github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1
github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE=
github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/royeo/dingrobot v1.0.1-0.20191230075228-c90a788ca8fd h1:YvxA3mges3/MbK3/tYEBprBeZhqSJOW82qAgHTiTBnM=
github.com/royeo/dingrobot v1.0.1-0.20191230075228-c90a788ca8fd/go.mod h1:RqDM8E/hySCVwI2aUFRJAUGDcHHRnIhzNmbNG3bamQs=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down
16 changes: 12 additions & 4 deletions log/hook/hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
stderr "errors"
"sync"
"time"

"github.com/Conflux-Chain/go-conflux-util/alert"
"github.com/pkg/errors"
Expand All @@ -25,6 +26,9 @@ type Config struct {
// Channels lists the default alert notification channels to use.
Channels []string

// Maximum request timeout allowed to send alert.
SendTimeout time.Duration `default:"3s"`

// Async configures the behavior of the asynchronous worker for handling log alerts.
Async AsyncOption
}
Expand Down Expand Up @@ -61,7 +65,7 @@ func AddAlertHook(ctx context.Context, wg *sync.WaitGroup, conf Config) error {
}

// Instantiate the base AlertHook.
var alertHook logrus.Hook = NewAlertHook(hookLvls, chs)
var alertHook logrus.Hook = NewAlertHook(hookLvls, chs, conf.SendTimeout)

// Wrap with asynchronous processing if configured.
if conf.Async.NumWorkers > 0 {
Expand Down Expand Up @@ -89,11 +93,12 @@ func wrapAsyncHook(
type AlertHook struct {
levels []logrus.Level
defaultChannels []alert.Channel
sendTimeout time.Duration
}

// NewAlertHook constructor to new AlertHook instance.
func NewAlertHook(lvls []logrus.Level, chs []alert.Channel) *AlertHook {
return &AlertHook{levels: lvls, defaultChannels: chs}
func NewAlertHook(lvls []logrus.Level, chs []alert.Channel, timeout time.Duration) *AlertHook {
return &AlertHook{levels: lvls, defaultChannels: chs, sendTimeout: timeout}
}

// implements `logrus.Hook` interface methods.
Expand All @@ -112,8 +117,11 @@ func (hook *AlertHook) Fire(logEntry *logrus.Entry) (err error) {
Content: logEntry,
}

ctx, cancel := context.WithTimeout(context.Background(), hook.sendTimeout)
defer cancel()

for _, ch := range notifyChans {
err = stderr.Join(ch.Send(note))
err = stderr.Join(ch.Send(ctx, note))
}

return errors.WithMessage(err, "failed to notify channel message")
Expand Down
3 changes: 2 additions & 1 deletion log/hook/hook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package hook
import (
"os"
"testing"
"time"

"github.com/Conflux-Chain/go-conflux-util/alert"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -52,7 +53,7 @@ func TestLogrusAddHooks(t *testing.T) {

// Add alert hook for logrus fatal/warn/error level
hookLevels := []logrus.Level{logrus.FatalLevel, logrus.WarnLevel, logrus.ErrorLevel}
logrus.AddHook(NewAlertHook(hookLevels, channels))
logrus.AddHook(NewAlertHook(hookLevels, channels, 3*time.Second))

// Need to manually check if message sent to dingtalk group chat
logrus.Warn("Test logrus add hooks warns")
Expand Down

0 comments on commit 6ed9a9b

Please sign in to comment.