diff --git a/plugins/out_clp_s3/Dockerfile b/plugins/out_clp_s3/Dockerfile index 692392f..adb2b0d 100644 --- a/plugins/out_clp_s3/Dockerfile +++ b/plugins/out_clp_s3/Dockerfile @@ -10,8 +10,7 @@ RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /bin WORKDIR /root -ENV GOOS=linux\ - GOARCH=amd64 +ENV GOOS=linux COPY / /root/ @@ -23,11 +22,11 @@ RUN task build FROM fluent/fluent-bit:4.2.2 -# Copy plugin binary to Fluent Bit image. +# Copy plugin binary to Fluent Bit image. COPY --from=builder /root/plugins/out_clp_s3/out_clp_s3.so /fluent-bit/bin/ -COPY --from=builder /root/plugins/out_clp_s3/*.conf /fluent-bit/etc/ +COPY --from=builder /root/plugins/out_clp_s3/fluent-bit.yaml /fluent-bit/etc/ # Port for listening interface for HTTP Server. EXPOSE 2020 -CMD ["/fluent-bit/bin/fluent-bit", "--config", "/fluent-bit/etc/fluent-bit.conf"] +CMD ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/fluent-bit.yaml", "-e", "/fluent-bit/bin/out_clp_s3.so"] diff --git a/plugins/out_clp_s3/README.md b/plugins/out_clp_s3/README.md index f3dbf59..bc356ff 100644 --- a/plugins/out_clp_s3/README.md +++ b/plugins/out_clp_s3/README.md @@ -7,24 +7,23 @@ Fluent Bit output plugin that sends records in CLP's compressed IR format to AWS First, confirm your AWS credentials are properly setup, see [AWS credentials](#AWS-credentials) for information. -Next, change [fluent-bit.conf](fluent-bit.conf) to suit your needs. Note, if your logs are JSON, you should use the [Fluent Bit JSON parser][1] on your input. +Next, change [fluent-bit.yaml](fluent-bit.yaml) to suit your needs. Note, if your logs are JSON, you should use the [Fluent Bit JSON parser][1] on your input. See [Plugin configuration](#plugin-configuration) for description of output options. See below for input and output examples: -``` -[INPUT] - name tail - path /var/log/app.json - tag app.json - parser basic_json -``` - -``` -[OUTPUT] - name out_clp_s3 - s3_bucket myBucket - match * +```yaml +pipeline: + inputs: + - name: tail + path: /var/log/app.json + tag: app.json + parser: json + + outputs: + - name: out_clp_s3 + match: "*" + s3_bucket: myBucket ``` Lastly start the plugin: @@ -41,14 +40,17 @@ First build the image Start a container ```shell - docker run -it -v ~/.aws/credentials:/root/.aws/credentials --rm fluent-bit-clp + docker run -it \ + -v ~/.aws/credentials:/root/.aws/credentials \ + -v ./fluent-bit.yaml:/fluent-bit/etc/fluent-bit.yaml \ + --rm fluent-bit-clp ``` Dummy logs will be written to your s3 bucket. #### Using local setup -Install [go][2] and [fluent-bit][3] +Install [go][2], [task][3], and [fluent-bit][4] Download go dependencies ```shell @@ -59,18 +61,10 @@ Run task to build a binary in the plugin directory ```shell task build ``` -Change [plugin-config.conf](plugin-config.conf) to reference the plugin binary - ```shell - [PLUGINS] - Path //out_clp_s3.so - ``` -Note changing this path may break docker setup. To preserve docker setup, copy -[plugin-config.conf](plugin-config.conf) and change `plugins_file` in -[fluent-bit.conf](fluent-bit.conf) to new file name. Run Fluent Bit ```shell - fluent-bit -c fluent-bit.conf + fluent-bit -e ./out_clp_s3.so -c fluent-bit.yaml ``` ### AWS Credentials @@ -80,13 +74,12 @@ The plugin will look for credentials using the following hierarchy: 3. If using ECS task definition or RunTask API, IAM role for tasks. 4. If running on an Amazon EC2 instance, IAM role for Amazon EC2. -Moreover, the plugin can assume a role by adding optional `role_arn` to -[plugin-config.conf](plugin-config.conf). Example shown below: -``` -role_arn arn:aws:iam::000000000000:role/accessToMyBucket +Moreover, the plugin can assume a role by adding optional `role_arn` to [fluent-bit.yaml](fluent-bit.yaml): +```yaml +role_arn: arn:aws:iam::000000000000:role/accessToMyBucket ``` -More detailed information for specifying credentials from AWS can be found [here][4]. +More detailed information for specifying credentials from AWS can be found [here][5]. ### Plugin configuration @@ -124,5 +117,6 @@ object using the tag key `fluentBitTag`. [1]: https://docs.fluentbit.io/manual/data-pipeline/parsers/json [2]: https://go.dev/doc/install -[3]: https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bit -[4]: https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials +[3]: https://taskfile.dev/installation +[4]: https://docs.fluentbit.io/manual/installation/getting-started-with-fluent-bit +[5]: https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html#specifying-credentials diff --git a/plugins/out_clp_s3/fluent-bit.conf b/plugins/out_clp_s3/fluent-bit.conf deleted file mode 100644 index 2555b48..0000000 --- a/plugins/out_clp_s3/fluent-bit.conf +++ /dev/null @@ -1,60 +0,0 @@ -#Sample Fluent Bit configuration with output set to CLP s3 plugin. - -[SERVICE] - # Flush - # ===== - # set an interval of seconds before to flush records to a destination - flush 1 - - # Daemon - # ====== - # instruct Fluent Bit to run in foreground or background mode. - daemon Off - - # Log_Level - # ========= - # Set the verbosity level of the service, values can be: - # - # - error - # - warning - # - info - # - debug - # - trace - # - # by default 'info' is set, that means it includes 'error' and 'warning'. - log_level info - - # Plugins File - # ============ - # specify an optional 'Plugins' configuration file to load external plugins. - plugins_file /fluent-bit/etc/plugin-config.conf - # plugins_file plugin-config.conf - parsers_file /fluent-bit/etc/parsers.conf - # parsers_file parsers.conf - - # HTTP Server - # =========== - # Enable/Disable the built-in HTTP Server for metrics. - http_server Off - http_listen 0.0.0.0 - http_port 2020 - -[INPUT] - name cpu - tag cpu.local - - # Read interval (sec) Default: 1 - interval_sec 1 - -# Example tail input with JSON parser -# [INPUT] -# name tail -# path /var/log/app.json -# tag app.json -# parser basic_json - -[OUTPUT] - name out_clp_s3 - s3_bucket myBucket - #role_arn arn:aws:iam::000000000000:role/accessToMyBucket - match * diff --git a/plugins/out_clp_s3/fluent-bit.yaml b/plugins/out_clp_s3/fluent-bit.yaml new file mode 100644 index 0000000..1027edc --- /dev/null +++ b/plugins/out_clp_s3/fluent-bit.yaml @@ -0,0 +1,26 @@ +# Sample Fluent Bit configuration with output set to CLP s3 plugin. +# Load plugin via CLI: fluent-bit -e ./out_clp_s3.so -c fluent-bit.yaml +--- + +parsers: + - name: json + format: json + +pipeline: + inputs: + # CPU outputs structured records, so no parser is needed + - name: cpu + tag: cpu.local + interval_sec: 1 + + # Example tail input with JSON parser + # - name: tail + # path: /var/log/app.json + # tag: app.json + # parser: json + + outputs: + - name: out_clp_s3 + match: "*" + s3_bucket: myBucket + # role_arn: arn:aws:iam::000000000000:role/accessToMyBucket diff --git a/plugins/out_clp_s3/parsers.conf b/plugins/out_clp_s3/parsers.conf deleted file mode 100644 index da5bc7e..0000000 --- a/plugins/out_clp_s3/parsers.conf +++ /dev/null @@ -1,3 +0,0 @@ -[PARSER] - Name basic_json - Format json diff --git a/plugins/out_clp_s3/plugin-config.conf b/plugins/out_clp_s3/plugin-config.conf deleted file mode 100644 index 43f24d8..0000000 --- a/plugins/out_clp_s3/plugin-config.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Plugin configuration referenced in Fluent Bit configuration -[PLUGINS] - path /fluent-bit/bin/out_clp_s3.so - #path ./out_clp_s3.so