Skip to content

Commit 1e5146a

Browse files
kikkomepkikkomep
authored
Merge pull request crs4#361 from kikkomep/feat/logs-export
feat/logging improvements
2 parents 77966a6 + a899d50 commit 1e5146a

20 files changed

+4122
-999
lines changed

‎Makefile

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,9 @@ lifemonitor: docker/lifemonitor.Dockerfile certs app.py gunicorn.conf.py ## Buil
139139
printf "\n$(yellow)WARNING: $(bold)Skip build of LifeMonitor Docker image !!! $(reset)\n" ; \
140140
else \
141141
printf "\n$(bold)Building LifeMonitor Docker image...$(reset)\n" ; \
142-
$(build_kit) docker $(build_cmd) $(cache_from_opt) $(cache_to_opt) \
142+
$(build_kit) docker $(build_cmd) \
143+
--build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \
144+
$(cache_from_opt) $(cache_to_opt) \
143145
${sw_version_arg} ${build_number_arg} ${tags_opt} ${labels_opt} ${platforms_opt} \
144146
-f docker/lifemonitor.Dockerfile -t crs4/lifemonitor . ;\
145147
printf "$(done)\n" ; \
@@ -221,14 +223,13 @@ start-dev: images compose-files dev reset_compose permissions ## Start LifeMonit
221223
start-testing: compose-files aux_images ro_crates images reset_compose permissions ## Start LifeMonitor in a Testing environment
222224
@printf "\n$(bold)Starting testing services...$(reset)\n" ; \
223225
base=$$(if [[ -f "docker-compose.yml" ]]; then echo "-f docker-compose.yml"; fi) ; \
224-
echo "$$(USER_UID=$$(id -u) USER_GID=$$(id -g) \
225-
$(docker_compose) $${base} \
226-
-f docker-compose.extra.yml \
227-
-f docker-compose.base.yml \
228-
-f docker-compose.monitoring.yml \
229-
-f docker-compose.dev.yml \
230-
-f docker-compose.test.yml \
231-
config)" > docker-compose.yml \
226+
echo "$$($(docker_compose) $${base} \
227+
-f docker-compose.extra.yml \
228+
-f docker-compose.base.yml \
229+
-f docker-compose.monitoring.yml \
230+
-f docker-compose.dev.yml \
231+
-f docker-compose.test.yml \
232+
config)" > docker-compose.yml \
232233
&& cp {,.test.}docker-compose.yml \
233234
&& $(docker_compose) -f docker-compose.yml up -d db lmtests seek jenkins webserver worker ws_server ;\
234235
$(docker_compose) -f ./docker-compose.yml \
@@ -262,7 +263,6 @@ start-aux-services: aux_images ro_crates docker-compose.extra.yml permissions ##
262263

263264
run-tests: start-testing ## Run all tests in the Testing Environment
264265
@printf "\n$(bold)Running tests...$(reset)\n" ; \
265-
USER_UID=$$(id -u) USER_GID=$$(id -g) \
266266
$(docker_compose) exec -T lmtests /bin/bash -c "pytest --durations=10 --color=yes tests"
267267

268268

‎docker/lifemonitor.Dockerfile

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,18 @@ RUN apt-get update -q \
88
postgresql-client-11 default-jre \
99
&& apt-get clean -y && rm -rf /var/lib/apt/lists
1010

11+
# Set the parametric USER ID
12+
ARG USER_ID
13+
ENV USER_ID=${USER_ID:-1000}
14+
15+
# Set the parametric GROUP ID
16+
ARG GROUP_ID
17+
ENV GROUP_ID=${GROUP_ID:-1000}
18+
1119
# Create a user 'lm' with HOME at /lm and set 'lm' as default git user
12-
RUN useradd -d /lm -m lm
20+
RUN groupadd -g ${GROUP_ID} lm && \
21+
useradd -u ${USER_ID} -g lm -d /lm -m lm
22+
1323
# Set the default user
1424
ENV USER=lm
1525

@@ -67,6 +77,7 @@ RUN mkdir -p /var/data/lm \
6777
&& chown -R lm:lm /var/data/lm \
6878
&& ln -s /var/data/lm /lm/data \
6979
&& chown -R lm:lm /lm/data \
80+
&& mkdir -p /var/log/lm && chown -R lm:lm /var/log/lm \
7081
&& mkdir /lm/.nextflow && chmod -R 777 /lm/.nextflow
7182

7283
# Set the default user

‎k8s/Chart.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ type: application
77
# This is the chart version. This version number should be incremented each time you make changes
88
# to the chart and its templates, including the app version.
99
# Versions are expected to follow Semantic Versioning (https://semver.org/)
10-
version: 0.10.0
10+
version: 0.11.0
1111

1212
# This is the version number of the application being deployed. This version number should be
1313
# incremented each time you make changes to the application. Versions are not expected to
1414
# follow Semantic Versioning. They should reflect the version the application is using.
15-
appVersion: 0.11.7
15+
appVersion: 0.11.8
1616

1717
# Chart dependencies
1818
dependencies:

‎k8s/templates/_helpers.tpl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ Define mount points shared by some pods.
167167
- mountPath: "/lm/certs/"
168168
name: lifemonitor-tls
169169
readOnly: true
170+
- name: lifemonitor-logs
171+
mountPath: "/var/log"
170172
- name: lifemonitor-settings
171173
mountPath: "/lm/settings.conf"
172174
subPath: settings.conf

‎k8s/templates/backend.deployment.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ spec:
5454
{{- include "lifemonitor.common-env" . | nindent 12 }}
5555
volumeMounts:
5656
{{- include "lifemonitor.common-volume-mounts" . | nindent 12 }}
57+
resources:
58+
{{- toYaml .Values.lifemonitor.initContainers.initBackend.resources | nindent 12 }}
5759
containers:
5860
- name: backend
5961
securityContext:
@@ -85,8 +87,27 @@ spec:
8587
periodSeconds: 3
8688
resources:
8789
{{- toYaml .Values.lifemonitor.resources | nindent 12 }}
90+
{{- if .Values.monitoring.loki.enabled }}
91+
- name: log-exporter
92+
image: {{ .Values.monitoring.loki.exporter.image }}
93+
imagePullPolicy: {{ $.Values.monitoring.loki.exporter.imagePullPolicy }}
94+
args:
95+
- "-config.file=/etc/promtail/promtail.yaml" # Found in the ConfigMap
96+
resources:
97+
{{- toYaml $.Values.monitoring.loki.exporter.resources | nindent 12 }}
98+
volumeMounts:
99+
- name: promtail-config
100+
mountPath: /etc/promtail
101+
- name: lifemonitor-logs
102+
mountPath: /var/log
103+
{{- end }}
88104
volumes:
89105
{{- include "lifemonitor.common-volume" . | nindent 8 }}
106+
{{- if .Values.monitoring.loki.enabled }}
107+
- name: promtail-config
108+
configMap:
109+
name: "{{.Release.Name}}-promtail-backend-configmap"
110+
{{- end }}
90111
{{- with .Values.lifemonitor.nodeSelector }}
91112
nodeSelector:
92113
{{- toYaml . | nindent 8 }}

‎k8s/templates/data-logs.pvc.yaml

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎k8s/templates/promtail.configmap.yaml

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
{{- if .Values.monitoring.loki.enabled }}
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: "{{$.Release.Name}}-promtail-proxy-configmap"
6+
data:
7+
promtail.yaml: |
8+
server:
9+
http_listen_port: 9080
10+
grpc_listen_port: 0
11+
log_level: "debug"
12+
positions:
13+
filename: /tmp/positions.yaml
14+
clients: # Specify target
15+
- url: {{ $.Values.monitoring.loki.url }}/loki/api/v1/push
16+
scrape_configs:
17+
- job_name: "lifemonitor-api-proxy-logger"
18+
static_configs:
19+
- targets:
20+
- localhost
21+
labels:
22+
app: "lifemonitor-backend"
23+
component: "{{$.Release.Name}}-proxy"
24+
environment: "{{$.Release.Namespace}}"
25+
format: "extended"
26+
level: "INFO"
27+
__path__: /var/log/nginx/access.log
28+
- targets:
29+
- localhost
30+
labels:
31+
app: "lifemonitor-backend"
32+
component: "{{ $.Release.Name }}-proxy"
33+
environment: "{{ $.Release.Namespace }}"
34+
format: "extended"
35+
level: "ERROR"
36+
__path__: /var/log/nginx/*error.log
37+
pipeline_stages:
38+
- drop:
39+
expression: ".*(DEBUG|health|heartbeat).*"
40+
---
41+
apiVersion: v1
42+
kind: ConfigMap
43+
metadata:
44+
name: "{{ $.Release.Name }}-promtail-backend-configmap"
45+
data:
46+
promtail.yaml: |
47+
server:
48+
http_listen_port: 9080
49+
grpc_listen_port: 0
50+
log_level: "debug"
51+
positions:
52+
filename: /tmp/positions.yaml
53+
clients: # Specify target
54+
- url: {{ $.Values.monitoring.loki.url }}/loki/api/v1/push
55+
scrape_configs:
56+
- job_name: "lifemonitor-api-backend-logger"
57+
static_configs:
58+
- targets:
59+
- localhost
60+
labels:
61+
app: "lifemonitor-backend"
62+
component: "api-backend"
63+
environment: "{{ $.Release.Namespace }}"
64+
format: "backend"
65+
__path__: /var/log/lm/*.log
66+
pipeline_stages:
67+
- drop:
68+
expression: ".*(DEBUG|health|heartbeat).*"
69+
- regex:
70+
expression: '(.*)(?P<log_level>ERROR|INFO|DEBUG|WARNING)(.*)'
71+
- labels:
72+
level: log_level
73+
74+
75+
{{- range $i, $queue := .Values.worker.queues }}
76+
---
77+
apiVersion: v1
78+
kind: ConfigMap
79+
metadata:
80+
name: "{{ $.Release.Name }}-promtail-worker-{{ $queue.name }}-configmap"
81+
data:
82+
promtail.yaml: |
83+
server:
84+
http_listen_port: 9080
85+
grpc_listen_port: 0
86+
log_level: "debug"
87+
positions:
88+
filename: /tmp/positions.yaml
89+
clients: # Specify target
90+
- url: {{ $.Values.monitoring.loki.url }}/loki/api/v1/push
91+
scrape_configs:
92+
- job_name: "lifemonitor-api-backend-logger"
93+
static_configs:
94+
- targets:
95+
- localhost
96+
labels:
97+
app: "lifemonitor-backend"
98+
component: "api-worker-{{ $queue.name }}"
99+
environment: "{{ $.Release.Namespace }}"
100+
format: "backend"
101+
__path__: /var/log/lm/*.log
102+
pipeline_stages:
103+
- drop:
104+
expression: ".*(DEBUG|health|heartbeat).*"
105+
- regex:
106+
expression: '(.*)(?P<log_level>ERROR|INFO|DEBUG|WARNING)(.*)'
107+
- labels:
108+
level: log_level
109+
{{- end }}
110+
{{- end }}

‎k8s/templates/worker.deployment.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ spec:
5050
{{- include "lifemonitor.common-env" $ | nindent 12 }}
5151
volumeMounts:
5252
{{- include "lifemonitor.common-volume-mounts" $ | nindent 12 }}
53+
resources:
54+
{{- toYaml $.Values.worker.initContainers.initWorker.resources | nindent 12 }}
5355
containers:
5456
- name: worker
5557
securityContext:
@@ -88,8 +90,27 @@ spec:
8890
# periodSeconds: 3
8991
resources:
9092
{{- toYaml $.Values.worker.resources | nindent 12 }}
93+
{{- if $.Values.monitoring.loki.enabled }}
94+
- name: log-exporter
95+
image: {{ $.Values.monitoring.loki.exporter.image }}
96+
imagePullPolicy: {{ $.Values.monitoring.loki.exporter.imagePullPolicy }}
97+
args:
98+
- "-config.file=/etc/promtail/promtail.yaml" # Found in the ConfigMap
99+
resources:
100+
{{- toYaml $.Values.monitoring.loki.exporter.resources | nindent 12 }}
101+
volumeMounts:
102+
- name: promtail-config
103+
mountPath: /etc/promtail
104+
- name: lifemonitor-logs
105+
mountPath: /var/log
106+
{{- end }}
91107
volumes:
92108
{{- include "lifemonitor.common-volume" $ | nindent 8 }}
109+
{{- if $.Values.monitoring.loki.enabled }}
110+
- name: promtail-config
111+
configMap:
112+
name: "{{$.Release.Name}}-promtail-worker-{{ $queue.name }}-configmap"
113+
{{- end }}
93114
{{- with $.Values.worker.nodeSelector }}
94115
nodeSelector:
95116
{{- toYaml . | nindent 8 }}

‎k8s/values.yaml

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,24 @@ monitoring:
136136
enabled: false
137137
prometheus:
138138
namespace: kube-prometheus-stack
139+
loki:
140+
enabled: false
141+
url: http://loki:3100
142+
exporter:
143+
image: grafana/promtail:main-60ea954
144+
imagePullPolicy: IfNotPresent
145+
resources:
146+
requests:
147+
memory: 128Mi
148+
cpu: 0.1
149+
limits:
150+
memory: 256Mi
151+
cpu: 0.2
139152

140153
rateLimiting:
141154
zone:
142155
accounts:
143-
enabled: false
156+
enabled: false
144157
size: 60m
145158
rate: 2r/s
146159
burst: 20
@@ -193,7 +206,6 @@ lifemonitor:
193206
enableTestConnection: false
194207

195208
resources:
196-
{}
197209
# We usually recommend not to specify default resources and to leave this as a conscious
198210
# choice for the user. This also increases chances charts run on environments with little
199211
# resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -204,6 +216,20 @@ lifemonitor:
204216
# requests:
205217
# cpu: 100m
206218
# memory: 128Mi
219+
limits:
220+
cpu: 1.8
221+
memory: 7936Mi
222+
requests:
223+
cpu: 0.5
224+
memory: 1024Mi
225+
226+
# configure resources for the init containers
227+
initContainers:
228+
initBackend:
229+
resources:
230+
limits:
231+
memory: 256Mi
232+
cpu: 200m
207233

208234
autoscaling:
209235
enabled: false
@@ -260,7 +286,6 @@ worker:
260286
replicaCount: 1
261287

262288
resources:
263-
{}
264289
# We usually recommend not to specify default resources and to leave this as a conscious
265290
# choice for the user. This also increases chances charts run on environments with little
266291
# resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -271,6 +296,20 @@ worker:
271296
# requests:
272297
# cpu: 100m
273298
# memory: 128Mi
299+
limits:
300+
cpu: 1.8
301+
memory: 7936Mi
302+
requests:
303+
cpu: 0.5
304+
memory: 1024Mi
305+
306+
# configure resources for the init containers
307+
initContainers:
308+
initWorker:
309+
resources:
310+
limits:
311+
memory: 256Mi
312+
cpu: 200m
274313

275314
autoscaling:
276315
enabled: false

0 commit comments

Comments
 (0)