Merge pull request #1 from mec07/initial

Basic synchronous implementation just to test that it works

.github/workflows/golangci-lint.yml

@@ -0,0 +1,27 @@
+name: golangci-lint
+on:
+  push:
+    tags:
+      - v*
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: test and lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14.x
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Coverage
+        run: go test -v -race -covermode=atomic -coverprofile=cover.out -timeout 10s ./...
+      - name: Report coverage
+        run: bash <(curl -s https://codecov.io/bash) -t 50f54c52-6302-41a7-a8f7-9835c21b53f6
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v1
+        with:
+          version: v1.27

CHANGELOG.md

@@ -0,0 +1,10 @@
+# Change Log
+
+All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.0.1] - 2020-08-12
+
+### Added
+
+- Basic synchronous implementation of NewWriter and NewWriterWithClient
+

README.md

@@ -1,2 +1,97 @@
 # zerolog2cloudwatch
-Package to enable sending logs from zerolog to AWS CloudWatch
+Package to enable sending logs from zerolog to AWS CloudWatch.
+
+## Usage
+
+This library assumes that you have IAM credentials to allow you to talk to AWS CloudWatch Logs.
+The specific permissions that are required are:
+- CreateLogGroup,
+- CreateLogStream,
+- DescribeLogStreams,
+- PutLogEvents.
+
+If these permissions aren't assigned to the user who's IAM credentials you're using then this package will not work.
+There are two exceptions to that:
+- if the log group already exists, then you don't need permission to CreateLogGroup;
+- if the log stream already exists, then you don't need permission to CreateLogStream.
+
+### Standard use case
+If you want zerolog to send all logs to CloudWatch then do the following:
+```
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/mec07/zerolog2cloudwatch"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+    region = "eu-west-2"
+    logGroupName = "log-group-name"
+    logStreamName = "log-stream-name"
+)
+
+func setupZerolog(accessKeyID, secretKey string) error {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewStaticCredentials(accessKeyID, secretKey, ""),
+	})
+	if err != nil {
+		return log.Logger, fmt.Errorf("session.NewSession: %w", err)
+	}
+
+	cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName)
+	if err != nil {
+		return log.Logger, fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err)
+	}
+
+	log.Logger = log.Output(cloudwatchWriter)
+}
+```
+If you prefer to use AWS IAM credentials that are saved in the usual location on your computer then you don't have to specify the credentials, e.g.:
+```
+sess, err := session.NewSession(&aws.Config{
+    Region:      aws.String(region),
+})
+```
+For more details, see: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/.
+See the example directory for a working example.
+
+### Write to CloudWatch and the console
+What I personally prefer is to write to both CloudWatch and the console, e.g.
+```
+cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName)
+if err != nil {
+    return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err)
+}
+consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout}
+log.Logger = log.Output(zerolog.MultiLevelWriter(consoleWriter, cloudwatchWriter))
+```
+
+### Create a new zerolog Logger
+Of course, you can create a new `zerolog.Logger` using this too:
+```
+cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName)
+if err != nil {
+    return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err)
+}
+logger := zerolog.New(cloudwatchWriter).With().Timestamp().Logger()
+```
+and of course you can create a new `zerolog.Logger` which can write to both CloudWatch and the console:
+```
+cloudwatchWriter, err := zerolog2cloudwatch.NewWriter(sess, logGroupName, logStreamName)
+if err != nil {
+    return fmt.Errorf("zerolog2cloudwatch.NewWriter: %w", err)
+}
+consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout}
+logger := zerolog.New(zerolog.MultiLevelWriter(consoleWriter, cloudwatchWriter)).With().Timestamp().Logger()
+```
+
+
+## Acknowledgements
+Much thanks has to go to the creator of `zerolog` (https://github.com/rs/zerolog), for creating such a good logger.
+Thanks must go to the writer of `logrus-cloudwatchlogs` (https://github.com/kdar/logrus-cloudwatchlogs) as I found it a helpful resource for interfacing with `cloudwatchlogs`.
+Thanks also goes to the writer of this: https://gist.github.com/asdine/f821abe6189a04250ae61b77a3048bd9, which I also found helpful for extracting logs from `zerolog`.

cloudwatch_writer.go

@@ -0,0 +1,121 @@
+package zerolog2cloudwatch
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/pkg/errors"
+)
+
+// CloudWatchLogsClient represents the AWS cloudwatchlogs client that we need to talk to CloudWatch
+type CloudWatchLogsClient interface {
+	DescribeLogStreams(*cloudwatchlogs.DescribeLogStreamsInput) (*cloudwatchlogs.DescribeLogStreamsOutput, error)
+	CreateLogGroup(*cloudwatchlogs.CreateLogGroupInput) (*cloudwatchlogs.CreateLogGroupOutput, error)
+	CreateLogStream(*cloudwatchlogs.CreateLogStreamInput) (*cloudwatchlogs.CreateLogStreamOutput, error)
+	PutLogEvents(*cloudwatchlogs.PutLogEventsInput) (*cloudwatchlogs.PutLogEventsOutput, error)
+}
+
+// CloudWatchWriter can be inserted into zerolog to send logs to CloudWatch.
+type CloudWatchWriter struct {
+	client        CloudWatchLogsClient
+	logGroupName  *string
+	logStreamName *string
+	sequenceToken *string
+}
+
+// NewWriter returns a pointer to a CloudWatchWriter struct, or an error.
+func NewWriter(sess *session.Session, logGroupName, logStreamName string) (*CloudWatchWriter, error) {
+	return NewWriterWithClient(cloudwatchlogs.New(sess), logGroupName, logStreamName)
+}
+
+// NewWriterWithClient returns a pointer to a CloudWatchWriter struct, or an error.
+func NewWriterWithClient(client CloudWatchLogsClient, logGroupName, logStreamName string) (*CloudWatchWriter, error) {
+	writer := &CloudWatchWriter{
+		client:        client,
+		logGroupName:  aws.String(logGroupName),
+		logStreamName: aws.String(logStreamName),
+	}
+
+	logStream, err := writer.getOrCreateLogStream()
+	if err != nil {
+		return nil, err
+	}
+
+	writer.sequenceToken = logStream.UploadSequenceToken
+
+	return writer, nil
+}
+
+// Write implements the io.Writer interface.
+func (c *CloudWatchWriter) Write(log []byte) (int, error) {
+	var logEvents []*cloudwatchlogs.InputLogEvent
+	logEvents = append(logEvents, &cloudwatchlogs.InputLogEvent{
+		Message: aws.String(string(log)),
+		// Timestamp has to be in milliseconds since the epoch
+		Timestamp: aws.Int64(time.Now().UTC().UnixNano() / int64(time.Millisecond)),
+	})
+
+	input := &cloudwatchlogs.PutLogEventsInput{
+		LogEvents:     logEvents,
+		LogGroupName:  c.logGroupName,
+		LogStreamName: c.logStreamName,
+		SequenceToken: c.sequenceToken,
+	}
+
+	resp, err := c.client.PutLogEvents(input)
+	if err != nil {
+		return 0, errors.Wrap(err, "cloudwatchlogs.Client.PutLogEvents")
+	}
+
+	if resp != nil {
+		c.sequenceToken = resp.NextSequenceToken
+	}
+
+	return len(log), nil
+}
+
+// getOrCreateLogStream gets info on the log stream for the log group and log
+// stream we're interested in -- primarily for the purpose of finding the value
+// of the next sequence token. If the log group doesn't exist, then we create
+// it, if the log stream doesn't exist, then we create it.
+func (c *CloudWatchWriter) getOrCreateLogStream() (*cloudwatchlogs.LogStream, error) {
+	// Get the log streams that match our log group name and log stream
+	output, err := c.client.DescribeLogStreams(&cloudwatchlogs.DescribeLogStreamsInput{
+		LogGroupName:        c.logGroupName,
+		LogStreamNamePrefix: c.logStreamName,
+	})
+	if err != nil || output == nil {
+		awserror, ok := err.(awserr.Error)
+		// i.e. the log group does not exist
+		if ok && awserror.Code() == cloudwatchlogs.ErrCodeResourceNotFoundException {
+			_, err = c.client.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
+				LogGroupName: c.logGroupName,
+			})
+			if err != nil {
+				return nil, errors.Wrap(err, "cloudwatchlog.Client.CreateLogGroup")
+			}
+			return c.getOrCreateLogStream()
+		}
+
+		return nil, errors.Wrap(err, "cloudwatchlogs.Client.DescribeLogStreams")
+	}
+
+	if len(output.LogStreams) > 0 {
+		return output.LogStreams[0], nil
+	}
+
+	// No matching log stream, so we need to create it
+	_, err = c.client.CreateLogStream(&cloudwatchlogs.CreateLogStreamInput{
+		LogGroupName:  c.logGroupName,
+		LogStreamName: c.logStreamName,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudwatchlogs.Client.CreateLogStream")
+	}
+
+	// We can just return an empty log stream as the initial sequence token would be nil anyway.
+	return &cloudwatchlogs.LogStream{}, nil
+}