From 1438aa85dd16af3e2fb134dea34fd69151bb02f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C4=8Cul=C3=A1k?= Date: Wed, 8 Nov 2023 13:19:58 +0100 Subject: [PATCH] Add fluentd Dockerfile 1.16.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michal Čulák --- fluentd-image/v1.15/Dockerfile | 111 ++++++++++++++++++++++++++++++ fluentd-image/v1.15/entrypoint.sh | 28 ++++++++ fluentd-image/v1.15/fluent.conf | 8 +++ fluentd-image/v1.15/healthy.sh | 23 +++++++ fluentd-image/v1.16/Dockerfile | 109 +++++++++++++++++++++++++++++ fluentd-image/v1.16/entrypoint.sh | 28 ++++++++ fluentd-image/v1.16/fluent.conf | 8 +++ fluentd-image/v1.16/healthy.sh | 23 +++++++ 8 files changed, 338 insertions(+) create mode 100644 fluentd-image/v1.15/Dockerfile create mode 100755 fluentd-image/v1.15/entrypoint.sh create mode 100644 fluentd-image/v1.15/fluent.conf create mode 100755 fluentd-image/v1.15/healthy.sh create mode 100644 fluentd-image/v1.16/Dockerfile create mode 100755 fluentd-image/v1.16/entrypoint.sh create mode 100644 fluentd-image/v1.16/fluent.conf create mode 100755 fluentd-image/v1.16/healthy.sh diff --git a/fluentd-image/v1.15/Dockerfile b/fluentd-image/v1.15/Dockerfile new file mode 100644 index 0000000000..3f25d395e5 --- /dev/null +++ b/fluentd-image/v1.15/Dockerfile @@ -0,0 +1,111 @@ +FROM alpine:3.16 +LABEL Description="Fluentd docker image" Vendor="Banzai Cloud" Version="1.15.3" + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apk delete' has no effect +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc \ + tini libmaxminddb geoip \ + && buildDeps=" \ + make gcc g++ libc-dev \ + wget bzip2 zlib-dev git linux-headers \ + automake autoconf libtool build-base \ + ruby-dev libc6-compat geoip-dev \ + " \ + && apk add --no-cache --virtual .build-deps \ + build-base \ + ruby-dev gnupg \ + && apk add $buildDeps \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.14.2 \ + && gem install http_parser.rb -v 0.5.3 \ + && gem install tzinfo -v 1.2.7 \ + && gem install resolv -v 0.2.1 \ + && gem install json -v 2.6.3 \ + && gem install async-http -v 0.56.3 \ +# && gem install ext_monitor \ + && gem install fluentd -v 1.15.3 \ + && gem install prometheus-client -v 2.1.0 \ + && gem install bigdecimal -v 1.4.4 \ + && gem install webrick \ + && gem install gelf -v 3.0.0 \ + # Pin the elasticsearch-ruby library < 7.14. + # See https://github.com/uken/fluent-plugin-elasticsearch/issues/912 + && gem install elasticsearch -v 7.13.3 \ + && gem install elasticsearch-xpack -v 7.13.3 \ + && gem install fluent-plugin-detect-exceptions -v 0.0.14 \ + # FMI https://github.com/banzaicloud/logging-operator/issues/887 \ + && find /usr/lib/ruby/gems/3.*/gems/fluent-plugin-detect-exceptions-* -type f -print0 | xargs -0 chmod 644 \ + && gem install \ + specific_install \ + fluent-plugin-remote-syslog \ + fluent-plugin-webhdfs \ + fluent-plugin-prometheus \ + fluent-plugin-s3 \ + fluent-plugin-rewrite-tag-filter \ + fluent-plugin-azure-storage-append-blob \ + fluent-plugin-oss \ + fluent-plugin-dedot_filter \ + fluent-plugin-sumologic_output \ + fluent-plugin-kafka \ + fluent-plugin-geoip \ +# fluent-plugin-label-router \ + fluent-plugin-tag-normaliser \ + fluent-plugin-grafana-loki \ + fluent-plugin-concat \ + fluent-plugin-kinesis \ + fluent-plugin-parser-logfmt \ + fluent-plugin-multi-format-parser \ + fluent-plugin-record-modifier \ + fluent-plugin-splunk-hec \ + fluent-plugin-newrelic \ + fluent-plugin-elasticsearch -v 5.2.5 \ + fluent-plugin-cloudwatch-logs \ + fluent-plugin-opensearch \ + fluent-plugin-throttle \ + fluent-plugin-logdna \ + fluent-plugin-datadog \ + fluent-plugin-aws-elasticsearch-service \ + fluent-plugin-redis \ + fluent-plugin-gelf-hs \ + fluent-plugin-sqs \ + fluent-plugin-kube-events-timestamp \ + fluent-plugin-grok-parser \ +# && gem specific_install -l https://github.com/tarokkk/fluent-plugin-logzio.git \ + && gem specific_install -l https://github.com/acquia/fluent-plugin-syslog_rfc5424.git \ + && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-gcs.git \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-metadata-filter \ + && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-label-router.git -b rwlock \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-enhance-k8s-metadata \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-sumologic \ + # https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/main/Dockerfile#L35 + && gem specific_install https://github.com/abonas/kubeclient --ref 831e360772c717aab5ca086521c45c86ee51435e \ + && apk del .build-deps $buildDeps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup -S fluent && adduser -S -G fluent fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd \ + && mkdir -p /buffers && chown -R fluent /buffers + + +COPY fluent.conf /fluentd/etc/ +COPY entrypoint.sh /bin/ +COPY healthy.sh /bin/ + + +ENV FLUENTD_CONF="fluent.conf" +#ENV RUBYLIB="/usr/lib/ruby/gems/2.7.0/gems/resolv-0.2.1/lib" + +ENV LD_PRELOAD="" +EXPOSE 24224 5140 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/fluentd-image/v1.15/entrypoint.sh b/fluentd-image/v1.15/entrypoint.sh new file mode 100755 index 0000000000..0850ff1687 --- /dev/null +++ b/fluentd-image/v1.15/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep ' \-c' ; then + set -- "$@" -c /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep ' \-p' ; then + set -- "$@" -p /fluentd/plugins + fi +fi + +exec "$@" \ No newline at end of file diff --git a/fluentd-image/v1.15/fluent.conf b/fluentd-image/v1.15/fluent.conf new file mode 100644 index 0000000000..04959f17a0 --- /dev/null +++ b/fluentd-image/v1.15/fluent.conf @@ -0,0 +1,8 @@ +# This is the root config file, which only includes components of the actual configuration + +# Do not collect fluentd's own logs to avoid infinite loops. + + @type null + + +@include /fluentd/etc/conf.d/*.conf diff --git a/fluentd-image/v1.15/healthy.sh b/fluentd-image/v1.15/healthy.sh new file mode 100755 index 0000000000..49587f9808 --- /dev/null +++ b/fluentd-image/v1.15/healthy.sh @@ -0,0 +1,23 @@ +#!/bin/sh -x + +# Liveness probe is aimed to help in situations where fluentd +# silently hangs for no apparent reasons until manual restart. +# The idea of this probe is that if fluentd is not queueing or +# flushing chunks for 5 minutes, something is not right. If +# you want to change the fluentd configuration, reducing amount of +# logs fluentd collects, consider changing the threshold or turning +# liveness probe off completely. +# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58 + +BUFFER_PATH=${BUFFER_PATH}; +LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; + +if [ ! -e ${BUFFER_PATH} ]; +then + exit 1; +fi; +touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness; +if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ]; +then + exit 1; +fi; diff --git a/fluentd-image/v1.16/Dockerfile b/fluentd-image/v1.16/Dockerfile new file mode 100644 index 0000000000..b0554725bd --- /dev/null +++ b/fluentd-image/v1.16/Dockerfile @@ -0,0 +1,109 @@ +FROM alpine:3.17 +LABEL Description="Fluentd docker image" Vendor="Banzai Cloud" Version="1.16.1" + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apk delete' has no effect +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc \ + tini libmaxminddb geoip \ + && buildDeps=" \ + make gcc g++ libc-dev \ + wget bzip2 zlib-dev git linux-headers \ + automake autoconf libtool build-base \ + ruby-dev libc6-compat geoip-dev \ + " \ + && apk add --no-cache --virtual .build-deps \ + build-base \ + ruby-dev gnupg \ + && apk add $buildDeps \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.14.2 \ + && gem install http_parser.rb -v 0.5.3 \ + && gem install tzinfo -v 1.2.7 \ + && gem install resolv -v 0.2.1 \ + && gem install json -v 2.6.3 \ + && gem install async-http -v 0.56.3 \ +# && gem install ext_monitor \ + && gem install fluentd -v 1.16.1 \ + && gem install prometheus-client -v 2.1.0 \ + && gem install bigdecimal -v 1.4.4 \ + && gem install webrick \ + && gem install gelf -v 3.0.0 \ + && gem install elasticsearch -v 8.7.0 \ + && gem install elasticsearch-xpack -v 7.13.3 \ + && gem install \ + specific_install \ + fluent-plugin-remote-syslog \ + fluent-plugin-webhdfs \ + fluent-plugin-prometheus \ + fluent-plugin-s3 \ + fluent-plugin-rewrite-tag-filter \ + fluent-plugin-azure-storage-append-blob \ + fluent-plugin-oss \ + fluent-plugin-dedot_filter \ + fluent-plugin-sumologic_output \ + fluent-plugin-kafka \ + fluent-plugin-geoip \ +# fluent-plugin-label-router \ + fluent-plugin-tag-normaliser \ + fluent-plugin-grafana-loki \ + fluent-plugin-concat \ + fluent-plugin-kinesis \ + fluent-plugin-parser-logfmt \ + fluent-plugin-multi-format-parser \ + fluent-plugin-record-modifier \ + fluent-plugin-splunk-hec \ + fluent-plugin-newrelic \ + fluent-plugin-elasticsearch \ + fluent-plugin-cloudwatch-logs \ + fluent-plugin-opensearch \ + fluent-plugin-throttle \ + fluent-plugin-logdna \ + fluent-plugin-datadog \ + fluent-plugin-aws-elasticsearch-service \ + fluent-plugin-redis \ + fluent-plugin-gelf-hs \ + fluent-plugin-sqs \ + fluent-plugin-kube-events-timestamp \ + fluent-plugin-grok-parser \ + # && gem specific_install -l https://github.com/tarokkk/fluent-plugin-logzio.git \ + # FMI https://github.com/banzaicloud/logging-operator/issues/887 \ + && gem specific_install -l https://github.com/kuldazbraslav/fluent-plugin-detect-exceptions.git \ + && find /usr/lib/ruby/gems/3.*/gems/fluent-plugin-detect-exceptions-* -type f -print0 | xargs -0 chmod 644 \ + && gem specific_install -l https://github.com/acquia/fluent-plugin-syslog_rfc5424.git \ + && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-gcs.git \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-metadata-filter \ + && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-label-router.git -b rwlock \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-enhance-k8s-metadata \ + && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-sumologic \ + # https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/main/Dockerfile#L35 + && gem specific_install https://github.com/abonas/kubeclient --ref 831e360772c717aab5ca086521c45c86ee51435e \ + && apk del .build-deps $buildDeps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup -S fluent && adduser -S -G fluent fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd \ + && mkdir -p /buffers && chown -R fluent /buffers + + +COPY fluent.conf /fluentd/etc/ +COPY entrypoint.sh /bin/ +COPY healthy.sh /bin/ + + +ENV FLUENTD_CONF="fluent.conf" +#ENV RUBYLIB="/usr/lib/ruby/gems/2.7.0/gems/resolv-0.2.1/lib" + +ENV LD_PRELOAD="" +EXPOSE 24224 5140 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/fluentd-image/v1.16/entrypoint.sh b/fluentd-image/v1.16/entrypoint.sh new file mode 100755 index 0000000000..0850ff1687 --- /dev/null +++ b/fluentd-image/v1.16/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep ' \-c' ; then + set -- "$@" -c /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep ' \-p' ; then + set -- "$@" -p /fluentd/plugins + fi +fi + +exec "$@" \ No newline at end of file diff --git a/fluentd-image/v1.16/fluent.conf b/fluentd-image/v1.16/fluent.conf new file mode 100644 index 0000000000..04959f17a0 --- /dev/null +++ b/fluentd-image/v1.16/fluent.conf @@ -0,0 +1,8 @@ +# This is the root config file, which only includes components of the actual configuration + +# Do not collect fluentd's own logs to avoid infinite loops. + + @type null + + +@include /fluentd/etc/conf.d/*.conf diff --git a/fluentd-image/v1.16/healthy.sh b/fluentd-image/v1.16/healthy.sh new file mode 100755 index 0000000000..49587f9808 --- /dev/null +++ b/fluentd-image/v1.16/healthy.sh @@ -0,0 +1,23 @@ +#!/bin/sh -x + +# Liveness probe is aimed to help in situations where fluentd +# silently hangs for no apparent reasons until manual restart. +# The idea of this probe is that if fluentd is not queueing or +# flushing chunks for 5 minutes, something is not right. If +# you want to change the fluentd configuration, reducing amount of +# logs fluentd collects, consider changing the threshold or turning +# liveness probe off completely. +# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58 + +BUFFER_PATH=${BUFFER_PATH}; +LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; + +if [ ! -e ${BUFFER_PATH} ]; +then + exit 1; +fi; +touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness; +if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ]; +then + exit 1; +fi;