diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..94acc07 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,27 @@ +name: golangci-lint +on: + push: + tags: + - v* + branches: + - master + pull_request: +jobs: + test: + name: test and lint + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.14.x + - name: Checkout code + uses: actions/checkout@v2 + - name: Coverage + run: go test -v -race -covermode=atomic -coverprofile=cover.out -timeout 10s ./... + - name: Report coverage + run: bash <(curl -s https://codecov.io/bash) -t 50f54c52-6302-41a7-a8f7-9835c21b53f6 + - name: golangci-lint + uses: golangci/golangci-lint-action@v1 + with: + version: v1.27 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f211238 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Change Log + +All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.0.1] - 2020-08-12 + +### Added + +- Basic synchronous implementation of NewWriter and NewWriterWithClient + diff --git a/README.md b/README.md index 32f5e9d..b72ab65 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,97 @@ # zerolog2cloudwatch -Package to enable sending logs from zerolog to AWS CloudWatch +Package to enable sending logs from zerolog to AWS CloudWatch. + +## Usage + +This library assumes that you have IAM credentials to allow you to talk to AWS CloudWatch Logs. +The specific permissions that are required are: +- CreateLogGroup, +- CreateLogStream, +- DescribeLogStreams, +- PutLogEvents. + +If these permissions aren't assigned to the user who's IAM credentials you're using then this package will not work. +There are two exceptions to that: +- if the log group already exists, then you don't need permission to CreateLogGroup; +- if the log stream already exists, then you don't need permission to CreateLogStream. + +### Standard use case +If you want zerolog to send all logs to CloudWatch then do the following: +``` +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/mec07/zerolog2cloudwatch" + "github.com/rs/zerolog/log" +) + +const ( + region = "eu-west-2" + logGroupName = "log-group-name" + logStreamName = "log-stream-name" +) + +func setupZerolog(accessKeyID, secretKey string) error { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(accessKeyID, secretKey, ""), + }) + if err != nil { + return log.Logger, fmt.Errorf("session.NewSession: %w", err) + } + + cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName) + if err != nil { + return log.Logger, fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err) + } + + log.Logger = log.Output(cloudwatchWriter) +} +``` +If you prefer to use AWS IAM credentials that are saved in the usual location on your computer then you don't have to specify the credentials, e.g.: +``` +sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), +}) +``` +For more details, see: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/. +See the example directory for a working example. + +### Write to CloudWatch and the console +What I personally prefer is to write to both CloudWatch and the console, e.g. +``` +cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName) +if err != nil { + return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err) +} +consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout} +log.Logger = log.Output(zerolog.MultiLevelWriter(consoleWriter, cloudwatchWriter)) +``` + +### Create a new zerolog Logger +Of course, you can create a new `zerolog.Logger` using this too: +``` +cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName) +if err != nil { + return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err) +} +logger := zerolog.New(cloudwatchWriter).With().Timestamp().Logger() +``` +and of course you can create a new `zerolog.Logger` which can write to both CloudWatch and the console: +``` +cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName) +if err != nil { + return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err) +} +consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout} +logger := zerolog.New(zerolog.MultiLevelWriter(consoleWriter, cloudwatchWriter)).With().Timestamp().Logger() +``` + + +## Acknowledgements +Much thanks has to go to the creator of `zerolog` (https://github.com/rs/zerolog), for creating such a good logger. +Thanks must go to the writer of `logrus-cloudwatchlogs` (https://github.com/kdar/logrus-cloudwatchlogs) as I found it a helpful resource for interfacing with `cloudwatchlogs`. +Thanks also goes to the writer of this: https://gist.github.com/asdine/f821abe6189a04250ae61b77a3048bd9, which I also found helpful for extracting logs from `zerolog`. \ No newline at end of file diff --git a/cloudwatch_writer.go b/cloudwatch_writer.go new file mode 100644 index 0000000..e6f89a3 --- /dev/null +++ b/cloudwatch_writer.go @@ -0,0 +1,121 @@ +package zerolog2cloudwatch + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/pkg/errors" +) + +// CloudWatchLogsClient represents the AWS cloudwatchlogs client that we need to talk to CloudWatch +type CloudWatchLogsClient interface { + DescribeLogStreams(*cloudwatchlogs.DescribeLogStreamsInput) (*cloudwatchlogs.DescribeLogStreamsOutput, error) + CreateLogGroup(*cloudwatchlogs.CreateLogGroupInput) (*cloudwatchlogs.CreateLogGroupOutput, error) + CreateLogStream(*cloudwatchlogs.CreateLogStreamInput) (*cloudwatchlogs.CreateLogStreamOutput, error) + PutLogEvents(*cloudwatchlogs.PutLogEventsInput) (*cloudwatchlogs.PutLogEventsOutput, error) +} + +// CloudWatchWriter can be inserted into zerolog to send logs to CloudWatch. +type CloudWatchWriter struct { + client CloudWatchLogsClient + logGroupName *string + logStreamName *string + sequenceToken *string +} + +// NewWriter returns a pointer to a CloudWatchWriter struct, or an error. +func NewWriter(sess *session.Session, logGroupName, logStreamName string) (*CloudWatchWriter, error) { + return NewWriterWithClient(cloudwatchlogs.New(sess), logGroupName, logStreamName) +} + +// NewWriterWithClient returns a pointer to a CloudWatchWriter struct, or an error. +func NewWriterWithClient(client CloudWatchLogsClient, logGroupName, logStreamName string) (*CloudWatchWriter, error) { + writer := &CloudWatchWriter{ + client: client, + logGroupName: aws.String(logGroupName), + logStreamName: aws.String(logStreamName), + } + + logStream, err := writer.getOrCreateLogStream() + if err != nil { + return nil, err + } + + writer.sequenceToken = logStream.UploadSequenceToken + + return writer, nil +} + +// Write implements the io.Writer interface. +func (c *CloudWatchWriter) Write(log []byte) (int, error) { + var logEvents []*cloudwatchlogs.InputLogEvent + logEvents = append(logEvents, &cloudwatchlogs.InputLogEvent{ + Message: aws.String(string(log)), + // Timestamp has to be in milliseconds since the epoch + Timestamp: aws.Int64(time.Now().UTC().UnixNano() / int64(time.Millisecond)), + }) + + input := &cloudwatchlogs.PutLogEventsInput{ + LogEvents: logEvents, + LogGroupName: c.logGroupName, + LogStreamName: c.logStreamName, + SequenceToken: c.sequenceToken, + } + + resp, err := c.client.PutLogEvents(input) + if err != nil { + return 0, errors.Wrap(err, "cloudwatchlogs.Client.PutLogEvents") + } + + if resp != nil { + c.sequenceToken = resp.NextSequenceToken + } + + return len(log), nil +} + +// getOrCreateLogStream gets info on the log stream for the log group and log +// stream we're interested in -- primarily for the purpose of finding the value +// of the next sequence token. If the log group doesn't exist, then we create +// it, if the log stream doesn't exist, then we create it. +func (c *CloudWatchWriter) getOrCreateLogStream() (*cloudwatchlogs.LogStream, error) { + // Get the log streams that match our log group name and log stream + output, err := c.client.DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{ + LogGroupName: c.logGroupName, + LogStreamNamePrefix: c.logStreamName, + }) + if err != nil || output == nil { + awserror, ok := err.(awserr.Error) + // i.e. the log group does not exist + if ok && awserror.Code() == cloudwatchlogs.ErrCodeResourceNotFoundException { + _, err = c.client.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ + LogGroupName: c.logGroupName, + }) + if err != nil { + return nil, errors.Wrap(err, "cloudwatchlog.Client.CreateLogGroup") + } + return c.getOrCreateLogStream() + } + + return nil, errors.Wrap(err, "cloudwatchlogs.Client.DescribeLogStreams") + } + + if len(output.LogStreams) > 0 { + return output.LogStreams[0], nil + } + + // No matching log stream, so we need to create it + _, err = c.client.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{ + LogGroupName: c.logGroupName, + LogStreamName: c.logStreamName, + }) + if err != nil { + return nil, errors.Wrap(err, "cloudwatchlogs.Client.CreateLogStream") + } + + // We can just return an empty log stream as the initial sequence token would be nil anyway. + return &cloudwatchlogs.LogStream{}, nil +} diff --git a/cloudwatch_writer_test.go b/cloudwatch_writer_test.go new file mode 100644 index 0000000..ea77a11 --- /dev/null +++ b/cloudwatch_writer_test.go @@ -0,0 +1,163 @@ +package zerolog2cloudwatch_test + +import ( + "encoding/json" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/mec07/zerolog2cloudwatch" + "github.com/stretchr/testify/assert" +) + +type mockClient struct { + sync.Mutex + logEvents []*cloudwatchlogs.InputLogEvent + logGroupName *string + logStreamName *string +} + +func (c *mockClient) DescribeLogStreams(*cloudwatchlogs.DescribeLogStreamsInput) (*cloudwatchlogs.DescribeLogStreamsOutput, error) { + c.Lock() + defer c.Unlock() + + if c.logGroupName == nil { + return nil, awserr.New(cloudwatchlogs.ErrCodeResourceNotFoundException, "blah", nil) + } + + var streams []*cloudwatchlogs.LogStream + if c.logStreamName != nil { + streams = append(streams, &cloudwatchlogs.LogStream{ + LogStreamName: c.logStreamName, + }) + } + + return &cloudwatchlogs.DescribeLogStreamsOutput{ + LogStreams: streams, + }, nil +} + +func (c *mockClient) CreateLogGroup(input *cloudwatchlogs.CreateLogGroupInput) (*cloudwatchlogs.CreateLogGroupOutput, error) { + c.Lock() + defer c.Unlock() + + c.logGroupName = input.LogGroupName + return nil, nil +} + +func (c *mockClient) CreateLogStream(input *cloudwatchlogs.CreateLogStreamInput) (*cloudwatchlogs.CreateLogStreamOutput, error) { + c.Lock() + defer c.Unlock() + + c.logStreamName = input.LogStreamName + return nil, nil +} + +func (c *mockClient) PutLogEvents(putLogEvents *cloudwatchlogs.PutLogEventsInput) (*cloudwatchlogs.PutLogEventsOutput, error) { + c.Lock() + defer c.Unlock() + + if putLogEvents == nil { + return nil, errors.New("received nil *cloudwatchlogs.PutLogEventsInput") + } + + c.logEvents = append(c.logEvents, putLogEvents.LogEvents...) + output := &cloudwatchlogs.PutLogEventsOutput{ + NextSequenceToken: aws.String("next sequence token"), + } + return output, nil +} + +func (c *mockClient) getLogEvents() []*cloudwatchlogs.InputLogEvent { + c.Lock() + defer c.Unlock() + + logEvents := make([]*cloudwatchlogs.InputLogEvent, len(c.logEvents)) + copy(logEvents, c.logEvents) + + return logEvents +} + +type exampleLog struct { + Time, Message, Filename string + Port uint16 +} + +func helperWriteLogs(t *testing.T, writer io.Writer, logs ...interface{}) { + for _, log := range logs { + message, err := json.Marshal(log) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + _, err = writer.Write(message) + if err != nil { + t.Fatalf("writer.Write(%s): %v", string(message), err) + } + } +} + +func helperMakeLogEvents(t *testing.T, logs ...interface{}) []*cloudwatchlogs.InputLogEvent { + var logEvents []*cloudwatchlogs.InputLogEvent + for _, log := range logs { + message, err := json.Marshal(log) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + logEvents = append(logEvents, &cloudwatchlogs.InputLogEvent{ + Message: aws.String(string(message)), + // Timestamps for CloudWatch Logs should be in milliseconds since the epoch. + Timestamp: aws.Int64(time.Now().UTC().UnixNano() / int64(time.Millisecond)), + }) + } + return logEvents +} + +// assertEqualLogMessages asserts that the log messages are all the same, ignoring the timestamps. +func assertEqualLogMessages(t *testing.T, expectedLogs []*cloudwatchlogs.InputLogEvent, logs []*cloudwatchlogs.InputLogEvent) { + if !assert.Equal(t, len(expectedLogs), len(logs), "expected to have the same number of logs") { + return + } + + for index, log := range logs { + if log == nil { + t.Fatalf("found nil log at index: %d", index) + } + if expectedLogs[index] == nil { + t.Fatalf("found nil log in expectedLogs at index: %d", index) + } + assert.Equal(t, *expectedLogs[index].Message, *log.Message) + } +} + +func TestCloudWatchWriter(t *testing.T) { + client := &mockClient{} + + cloudWatchWriter, err := zerolog2cloudwatch.NewWriterWithClient(client, "logGroup", "logStream") + if err != nil { + t.Fatalf("NewWriterWithClient: %v", err) + } + + aLog1 := exampleLog{ + Time: "2009-11-10T23:00:02.043123061Z", + Message: "Test message 1", + Filename: "filename", + Port: 666, + } + aLog2 := exampleLog{ + Time: "2009-11-10T23:00:02.043123061Z", + Message: "Test message 2", + Filename: "filename", + Port: 666, + } + + helperWriteLogs(t, cloudWatchWriter, aLog1, aLog2) + + expectedLogs := helperMakeLogEvents(t, aLog1, aLog2) + + assertEqualLogMessages(t, expectedLogs, client.getLogEvents()) +} diff --git a/example/example.go b/example/example.go new file mode 100644 index 0000000..51026cc --- /dev/null +++ b/example/example.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/mec07/zerolog2cloudwatch" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +const ( + region = "eu-west-2" + logGroupName = "zerolog2cloudwatch" + logStreamName = "this-computer" +) + +func main() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretKey := os.Getenv("SECRET_ACCESS_KEY") + + logger, err := newLogger(accessKeyID, secretKey) + if err != nil { + log.Error().Err(err).Msg("newLogger") + return + } + + logger.Info().Str("name", "zerolog2cloudwatch").Msg("Log to test out this package") +} + +func newLogger(accessKeyID, secretKey string) (zerolog.Logger, error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(accessKeyID, secretKey, ""), + }) + if err != nil { + return log.Logger, fmt.Errorf("session.NewSession: %w", err) + } + + cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName) + if err != nil { + return log.Logger, fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err) + } + + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout} + logger := zerolog.New(zerolog.MultiLevelWriter(consoleWriter, cloudwatchWriter)).With().Timestamp().Logger() + + return logger, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a9c4fc6 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/mec07/zerolog2cloudwatch + +go 1.14 + +require ( + github.com/aws/aws-sdk-go v1.34.2 + github.com/pkg/errors v0.9.1 + github.com/rs/zerolog v1.19.0 + github.com/stretchr/testify v1.6.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bf2d415 --- /dev/null +++ b/go.sum @@ -0,0 +1,35 @@ +github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= +github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg= +github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=