Add fluentd Dockerfile 1.16.2

Signed-off-by: Michal Čulák <michal.culak@jamf.com>
wandera · Nov 8, 2023 · 1438aa8 · 1438aa8
1 parent 697efff
commit 1438aa8
Show file tree

Hide file tree

Showing 8 changed files with 338 additions and 0 deletions.
diff --git a/fluentd-image/v1.15/Dockerfile b/fluentd-image/v1.15/Dockerfile
@@ -0,0 +1,111 @@
+FROM alpine:3.16
+LABEL Description="Fluentd docker image" Vendor="Banzai Cloud" Version="1.15.3"
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apk delete' has no effect
+RUN apk update \
+ && apk add --no-cache \
+        ca-certificates \
+        ruby ruby-irb ruby-etc \
+        tini libmaxminddb geoip \
+ && buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 zlib-dev git linux-headers \
+      automake autoconf libtool build-base \
+      ruby-dev libc6-compat geoip-dev \
+    " \
+ && apk add --no-cache --virtual .build-deps \
+        build-base \
+        ruby-dev gnupg \
+ && apk add $buildDeps \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ && gem install oj -v 3.14.2 \
+ && gem install http_parser.rb -v 0.5.3 \
+ && gem install tzinfo -v 1.2.7 \
+ && gem install resolv -v 0.2.1 \
+ && gem install json -v 2.6.3 \
+ && gem install async-http -v 0.56.3 \
+# && gem install ext_monitor \
+ && gem install fluentd -v 1.15.3 \
+ && gem install prometheus-client -v 2.1.0 \
+ && gem install bigdecimal -v 1.4.4 \
+ && gem install webrick \
+ && gem install gelf -v 3.0.0 \
+ # Pin the elasticsearch-ruby library < 7.14.
+ # See https://github.com/uken/fluent-plugin-elasticsearch/issues/912
+ && gem install elasticsearch -v 7.13.3 \
+ && gem install elasticsearch-xpack -v 7.13.3 \
+ && gem install fluent-plugin-detect-exceptions -v 0.0.14 \
+ # FMI https://github.com/banzaicloud/logging-operator/issues/887 \
+ && find /usr/lib/ruby/gems/3.*/gems/fluent-plugin-detect-exceptions-* -type f -print0 | xargs -0 chmod 644 \
+ && gem install \
+         specific_install \
+         fluent-plugin-remote-syslog \
+         fluent-plugin-webhdfs \
+         fluent-plugin-prometheus \
+         fluent-plugin-s3 \
+         fluent-plugin-rewrite-tag-filter \
+         fluent-plugin-azure-storage-append-blob \
+         fluent-plugin-oss \
+         fluent-plugin-dedot_filter \
+         fluent-plugin-sumologic_output \
+         fluent-plugin-kafka \
+         fluent-plugin-geoip \
+#         fluent-plugin-label-router \
+         fluent-plugin-tag-normaliser \
+         fluent-plugin-grafana-loki \
+         fluent-plugin-concat \
+         fluent-plugin-kinesis \
+         fluent-plugin-parser-logfmt \
+         fluent-plugin-multi-format-parser \
+         fluent-plugin-record-modifier \
+         fluent-plugin-splunk-hec \
+         fluent-plugin-newrelic \
+         fluent-plugin-elasticsearch -v 5.2.5 \
+         fluent-plugin-cloudwatch-logs \
+         fluent-plugin-opensearch \
+         fluent-plugin-throttle \
+         fluent-plugin-logdna \
+         fluent-plugin-datadog \
+         fluent-plugin-aws-elasticsearch-service \
+         fluent-plugin-redis \
+         fluent-plugin-gelf-hs \
+         fluent-plugin-sqs \
+         fluent-plugin-kube-events-timestamp \
+         fluent-plugin-grok-parser \
+# && gem specific_install -l https://github.com/tarokkk/fluent-plugin-logzio.git \
+ && gem specific_install -l https://github.com/acquia/fluent-plugin-syslog_rfc5424.git \
+ && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-gcs.git \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-metadata-filter \
+ && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-label-router.git -b rwlock \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-enhance-k8s-metadata \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-sumologic \
+ # https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/main/Dockerfile#L35
+ && gem specific_install https://github.com/abonas/kubeclient --ref 831e360772c717aab5ca086521c45c86ee51435e \
+ && apk del .build-deps $buildDeps \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test
+
+RUN addgroup -S fluent && adduser -S -G fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd \
+    && mkdir -p /buffers && chown -R fluent /buffers
+
+
+COPY fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+COPY healthy.sh /bin/
+
+
+ENV FLUENTD_CONF="fluent.conf"
+#ENV RUBYLIB="/usr/lib/ruby/gems/2.7.0/gems/resolv-0.2.1/lib"
+
+ENV LD_PRELOAD=""
+EXPOSE 24224 5140
+
+USER fluent
+ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
+CMD ["fluentd"]
diff --git a/fluentd-image/v1.15/entrypoint.sh b/fluentd-image/v1.15/entrypoint.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep ' \-c' ; then
+       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep ' \-p' ; then
+       set -- "$@" -p /fluentd/plugins
+    fi
+fi
+
+exec "$@"
diff --git a/fluentd-image/v1.15/fluent.conf b/fluentd-image/v1.15/fluent.conf
@@ -0,0 +1,8 @@
+# This is the root config file, which only includes components of the actual configuration
+
+# Do not collect fluentd's own logs to avoid infinite loops.
+<match fluent.**>
+  @type null
+</match>
+
+@include /fluentd/etc/conf.d/*.conf
diff --git a/fluentd-image/v1.15/healthy.sh b/fluentd-image/v1.15/healthy.sh
@@ -0,0 +1,23 @@
+#!/bin/sh -x
+
+# Liveness probe is aimed to help in situations where fluentd
+# silently hangs for no apparent reasons until manual restart.
+# The idea of this probe is that if fluentd is not queueing or
+# flushing chunks for 5 minutes, something is not right. If
+# you want to change the fluentd configuration, reducing amount of
+# logs fluentd collects, consider changing the threshold or turning
+# liveness probe off completely.
+# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58
+
+BUFFER_PATH=${BUFFER_PATH};
+LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
+
+if [ ! -e ${BUFFER_PATH} ];
+then
+  exit 1;
+fi;
+touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness;
+if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ];
+then
+  exit 1;
+fi;
diff --git a/fluentd-image/v1.16/Dockerfile b/fluentd-image/v1.16/Dockerfile
@@ -0,0 +1,109 @@
+FROM alpine:3.17
+LABEL Description="Fluentd docker image" Vendor="Banzai Cloud" Version="1.16.1"
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+# therefore an 'apk delete' has no effect
+RUN apk update \
+ && apk add --no-cache \
+        ca-certificates \
+        ruby ruby-irb ruby-etc \
+        tini libmaxminddb geoip \
+ && buildDeps=" \
+      make gcc g++ libc-dev \
+      wget bzip2 zlib-dev git linux-headers \
+      automake autoconf libtool build-base \
+      ruby-dev libc6-compat geoip-dev \
+    " \
+ && apk add --no-cache --virtual .build-deps \
+        build-base \
+        ruby-dev gnupg \
+ && apk add $buildDeps \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ && gem install oj -v 3.14.2 \
+ && gem install http_parser.rb -v 0.5.3 \
+ && gem install tzinfo -v 1.2.7 \
+ && gem install resolv -v 0.2.1 \
+ && gem install json -v 2.6.3 \
+ && gem install async-http -v 0.56.3 \
+# && gem install ext_monitor \
+ && gem install fluentd -v 1.16.1 \
+ && gem install prometheus-client -v 2.1.0 \
+ && gem install bigdecimal -v 1.4.4 \
+ && gem install webrick \
+ && gem install gelf -v 3.0.0 \
+ && gem install elasticsearch -v 8.7.0 \
+ && gem install elasticsearch-xpack -v 7.13.3 \
+ && gem install \
+         specific_install \
+         fluent-plugin-remote-syslog \
+         fluent-plugin-webhdfs \
+         fluent-plugin-prometheus \
+         fluent-plugin-s3 \
+         fluent-plugin-rewrite-tag-filter \
+         fluent-plugin-azure-storage-append-blob \
+         fluent-plugin-oss \
+         fluent-plugin-dedot_filter \
+         fluent-plugin-sumologic_output \
+         fluent-plugin-kafka \
+         fluent-plugin-geoip \
+#         fluent-plugin-label-router \
+         fluent-plugin-tag-normaliser \
+         fluent-plugin-grafana-loki \
+         fluent-plugin-concat \
+         fluent-plugin-kinesis \
+         fluent-plugin-parser-logfmt \
+         fluent-plugin-multi-format-parser \
+         fluent-plugin-record-modifier \
+         fluent-plugin-splunk-hec \
+         fluent-plugin-newrelic \
+         fluent-plugin-elasticsearch \
+         fluent-plugin-cloudwatch-logs \
+         fluent-plugin-opensearch \
+         fluent-plugin-throttle \
+         fluent-plugin-logdna \
+         fluent-plugin-datadog \
+         fluent-plugin-aws-elasticsearch-service \
+         fluent-plugin-redis \
+         fluent-plugin-gelf-hs \
+         fluent-plugin-sqs \
+         fluent-plugin-kube-events-timestamp \
+         fluent-plugin-grok-parser \
+ # && gem specific_install -l https://github.com/tarokkk/fluent-plugin-logzio.git \
+ # FMI https://github.com/banzaicloud/logging-operator/issues/887 \
+ && gem specific_install -l https://github.com/kuldazbraslav/fluent-plugin-detect-exceptions.git \
+ && find /usr/lib/ruby/gems/3.*/gems/fluent-plugin-detect-exceptions-* -type f -print0 | xargs -0 chmod 644 \
+ && gem specific_install -l https://github.com/acquia/fluent-plugin-syslog_rfc5424.git \
+ && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-gcs.git \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-metadata-filter \
+ && gem specific_install -l https://github.com/banzaicloud/fluent-plugin-label-router.git -b rwlock \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-enhance-k8s-metadata \
+ && gem specific_install -l https://github.com/SumoLogic/sumologic-kubernetes-fluentd.git -d fluent-plugin-kubernetes-sumologic \
+ # https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/main/Dockerfile#L35
+ && gem specific_install https://github.com/abonas/kubeclient --ref 831e360772c717aab5ca086521c45c86ee51435e \
+ && apk del .build-deps $buildDeps \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test
+
+RUN addgroup -S fluent && adduser -S -G fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd \
+    && mkdir -p /buffers && chown -R fluent /buffers
+
+
+COPY fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+COPY healthy.sh /bin/
+
+
+ENV FLUENTD_CONF="fluent.conf"
+#ENV RUBYLIB="/usr/lib/ruby/gems/2.7.0/gems/resolv-0.2.1/lib"
+
+ENV LD_PRELOAD=""
+EXPOSE 24224 5140
+
+USER fluent
+ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
+CMD ["fluentd"]
diff --git a/fluentd-image/v1.16/entrypoint.sh b/fluentd-image/v1.16/entrypoint.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep ' \-c' ; then
+       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep ' \-p' ; then
+       set -- "$@" -p /fluentd/plugins
+    fi
+fi
+
+exec "$@"
diff --git a/fluentd-image/v1.16/fluent.conf b/fluentd-image/v1.16/fluent.conf
@@ -0,0 +1,8 @@
+# This is the root config file, which only includes components of the actual configuration
+
+# Do not collect fluentd's own logs to avoid infinite loops.
+<match fluent.**>
+  @type null
+</match>
+
+@include /fluentd/etc/conf.d/*.conf
diff --git a/fluentd-image/v1.16/healthy.sh b/fluentd-image/v1.16/healthy.sh
@@ -0,0 +1,23 @@
+#!/bin/sh -x
+
+# Liveness probe is aimed to help in situations where fluentd
+# silently hangs for no apparent reasons until manual restart.
+# The idea of this probe is that if fluentd is not queueing or
+# flushing chunks for 5 minutes, something is not right. If
+# you want to change the fluentd configuration, reducing amount of
+# logs fluentd collects, consider changing the threshold or turning
+# liveness probe off completely.
+# soiurce https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml#L58
+
+BUFFER_PATH=${BUFFER_PATH};
+LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300};
+
+if [ ! -e ${BUFFER_PATH} ];
+then
+  exit 1;
+fi;
+touch -d "@$(($(date +%s) - $LIVENESS_THRESHOLD_SECONDS))" /tmp/marker-liveness;
+if [ -z "$(find ${BUFFER_PATH} -type d -newer /tmp/marker-liveness -print -quit)" ];
+then
+  exit 1;
+fi;