From 1ab6e170a1403f69e85c6746c3ef44e367e7e665 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 10 Jun 2024 02:11:20 +0200 Subject: [PATCH] integrate wis2-downloader in wis2box-stack (#688) * integrate wis2-downloader in wis2box-stack * add documentation * fix messed up entrypoint * prometheus scraping plus grafana download dashboard * flake8 clean.py --- .github/workflows/tests-docker.yml | 2 +- docker-compose.monitoring.yml | 2 + docker-compose.subscribe-download.yml | 10 - docker-compose.yml | 8 + docs/source/user/downloading-data.rst | 101 +-- grafana/dashboards/wis2-downloader.json | 870 ++++++++++++++++++++++++ nginx/nginx-ssl.conf | 7 + nginx/nginx.conf | 7 + prometheus/prometheus.yml | 5 + wis2-downloader/Dockerfile | 46 ++ wis2-downloader/clean.cron | 1 + wis2-downloader/clean.py | 39 ++ wis2-downloader/config.template | 15 + wis2-downloader/entrypoint.sh | 31 + wis2box-management/docker/entrypoint.sh | 9 + wis2box-subscribe-download/Dockerfile | 30 - wis2box-subscribe-download/local.yml | 14 - 17 files changed, 1099 insertions(+), 98 deletions(-) delete mode 100644 docker-compose.subscribe-download.yml create mode 100644 grafana/dashboards/wis2-downloader.json create mode 100644 wis2-downloader/Dockerfile create mode 100644 wis2-downloader/clean.cron create mode 100644 wis2-downloader/clean.py create mode 100644 wis2-downloader/config.template create mode 100644 wis2-downloader/entrypoint.sh delete mode 100644 wis2box-subscribe-download/Dockerfile delete mode 100644 wis2box-subscribe-download/local.yml diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 621dc6db..e10959e4 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -42,7 +42,7 @@ jobs: docker logs wis2box-management - name: setup wis2box-management ⚙️ run: | - sleep 20 + sleep 30 python3 wis2box-ctl.py execute wis2box environment show - name: populate stations from CSV 📡 run: | diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 0ec59f04..c7f0dc64 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -114,6 +114,8 @@ services: <<: *logging web-proxy: <<: *logging + wis2-downloader: + <<: *logging volumes: loki-data: diff --git a/docker-compose.subscribe-download.yml b/docker-compose.subscribe-download.yml deleted file mode 100644 index 873b4ea1..00000000 --- a/docker-compose.subscribe-download.yml +++ /dev/null @@ -1,10 +0,0 @@ -services: - wis2box-api: - container_name: wis2box-subscribe-download - build: - context: wis2box-subscribe-download - volumes: - - subscribe-download:/data/wis2box-subscribe-download - -volumes: - subscribe-download: diff --git a/docker-compose.yml b/docker-compose.yml index 2adab77b..4afb92b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,6 +145,14 @@ services: depends_on: - wis2box-management + wis2-downloader: + container_name: wis2-downloader + build: ./wis2-downloader + env_file: + - wis2box.env + volumes: + - ${WIS2BOX_HOST_DATADIR}/downloads:/app/downloads:rw + volumes: es-data: minio-data: diff --git a/docs/source/user/downloading-data.rst b/docs/source/user/downloading-data.rst index d8e1246b..e3cadcde 100644 --- a/docs/source/user/downloading-data.rst +++ b/docs/source/user/downloading-data.rst @@ -11,71 +11,86 @@ This section provides guidance how to download data from WIS2 Global Services. WIS2 Global Services include a Global Broker that provides users the ability to subscribe to data (via topics) and download to their local environment / workstation / decision support system from the WIS2 Global Cache. -The pywis-pubsub tool ---------------------- +wis2-downloader +--------------- -wis2box enables subscribe and data download workflow the WIS2 network, by using the ``wis2box-subscribe-download`` container, inside of which runs the `pywis-pubsub tool `_ +wis2box enables subscribe and data download workflow the WIS2 network, by using the ``wis2-downloader`` container, inside of which runs the `wis2-downloader tool " +``` - docker compose -f docker.subscribe-download.yml down +The list of the currently active subscriptions should be returned as a JSON object. +Adding subscriptions +~~~~~~~~~~~~~~~~~~~~ -Running pywis-pubsub interactively ----------------------------------- +Subscriptions can be added via a GET request to the `./add` endpoint that is proxied on /wis2-downloader on the wis2box host, with the following form: -pywis-pubsub can also be run interactively from inside the wis2box main container as follows: +```bash +curl http://localhost/wis2-downloader/add?topic=&target= -H "Authorization: Bearer " +``` -.. code-block:: bash +- `topic` specifies the topic to subscribe to. *Special characters (+, #) must be URL encoded, i.e. `+` = `%2B`, `#` = `%23`.* +- `target` specifies the directory to save the downloads to, relative to `download_dir` from `config.json`. *If this is not provided, the directory will default to that of the topic hierarchy.* - # login to wis2box main container - python3 wis2box-ctl.py login +For example: +```bash +curl http://localhost/wis2-downloader/add?topic=cache/a/wis2/%2B/data/core/weather/%23&target=example_data -H "Authorization: Bearer " +``` - # edit a local configuration by using wis2box-subscribe-download/local.yml as a template - vi /data/wis2box/local.yml +The list of active subscriptions after addition should be returned as a JSON object. + +Deleting subscriptions +~~~~~~~~~~~~~~~~~~~~~~ + +Subscriptions are deleted similarly via a GET request to the `./delete` endpoint, with the following form: +```bash +curl http://:/delete?topic= -H "Authorization: Bearer " +``` + +For example: +```bash +curl http://localhost:8080/delete?topic=cache/a/wis2/%2B/data/core/weather/%23 -H "Authorization: Bearer " +``` + +The list of active subscriptions after deletion should be returned as a JSON object. - # connect, and simply display data notifications - pywis-pubsub subscribe --config local.yml - # connect, and additionally download messages - pywis-pubsub subscribe --config local.yml --download - # connect, and filter messages by bounding box geometry - pywis-pubsub subscribe --config local.yml --bbox=-142,42,-52,84 diff --git a/grafana/dashboards/wis2-downloader.json b/grafana/dashboards/wis2-downloader.json new file mode 100644 index 00000000..a6136bcf --- /dev/null +++ b/grafana/dashboards/wis2-downloader.json @@ -0,0 +1,870 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Topic" + } + ] + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(topic) (topic_subscription_status == 1)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "WIS2-downloader subscriptions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "topic": "Topic" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 5, + "y": 0 + }, + "id": 18, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "expr": "queue_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Download queue size", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "fixed" + }, + "mappings": [], + "max": 1000, + "min": 0, + "noValue": "No data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 8, + "y": 0 + }, + "id": 11, + "interval": "1h", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_files_total[1h]))", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "{{label_name}}", + "range": false, + "refId": "A" + } + ], + "title": "Files downloaded last hour", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "fixed" + }, + "decimals": 1, + "mappings": [], + "max": 1000, + "min": 0, + "noValue": "No data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 100 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 11, + "y": 0 + }, + "id": 12, + "interval": "1h", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_bytes_total[1h]))", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "{{label_name}}", + "range": false, + "refId": "A" + } + ], + "title": "Data downloaded last hour", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "fixed" + }, + "mappings": [], + "max": 9997, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 0 + }, + "id": 16, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_files_total{file_type=\"bufr\"}[1h]))", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_files_total{file_type=\"grib\"}[1h]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(downloaded_files_total{file_type!~\"bufr|grib\"}[1h]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Files downloaded by file-type", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "fixed" + }, + "decimals": 1, + "mappings": [], + "max": 1000000000, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 19, + "interval": "1h", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + "titleSize": 17, + "valueSize": 20 + } + }, + "pluginVersion": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_bytes_total{file_type=\"bufr\"}[1h]))", + "format": "time_series", + "instant": true, + "interval": "1h", + "legendFormat": "BUFR", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum(increase(downloaded_bytes_total{file_type=\"grib\"}[1h]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "GRIB", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(downloaded_bytes_total{file_type!~\"grib|bufr\"}[1h]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "1h", + "legendFormat": "Other", + "range": false, + "refId": "C" + } + ], + "title": "Data downloaded by file-type", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 19, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Data downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Data downloaded per minute" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Data downloaded per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 14, + "x": 0, + "y": 4 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "expr": "sum(increase(downloaded_bytes_total[1m]))", + "hide": false, + "legendFormat": "Data downloaded per minute", + "range": true, + "refId": "A" + } + ], + "title": "Data downloaded per minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P982945308D3682D1" + }, + "gridPos": { + "h": 18, + "w": 10, + "x": 14, + "y": 4 + }, + "id": 6, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P982945308D3682D1" + }, + "editorMode": "builder", + "expr": "{container_name=\"wis2-downloader\"} != `Queue` != `GET` != `CLEAN`", + "queryType": "range", + "refId": "A" + } + ], + "title": "WIS2 downloader logs", + "transparent": true, + "type": "logs" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "download per minute" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Files downloaded per hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 14, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P1809F7CD0C75ACF3" + }, + "editorMode": "builder", + "expr": "sum(increase(downloaded_files_total[1m]))", + "legendFormat": "Files downloaded per minute", + "range": true, + "refId": "A" + } + ], + "title": "Files downloaded per minute", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "wis2-downloader monitoring dashboard", + "uid": "zB4GjVaIk", + "version": 5, + "weekStart": "" + } \ No newline at end of file diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 1500aca1..64f9a044 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -59,6 +59,13 @@ rewrite ^/data(/.*)$ /wis2box-public$1 break; proxy_pass http://minio:9000; } + location /wis2-downloader { + set $x_api_http_method $request_method; + auth_request /auth; + auth_request_set $auth_status $upstream_status; + rewrite ^/wis2-downloader/(.*) /$1 break; + proxy_pass http://wis2-downloader:5000; + } location /oapi { set $x_api_http_method $request_method; auth_request /auth; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index da76934d..362e5887 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -62,6 +62,13 @@ auth_request_set $auth_status $upstream_status; proxy_pass http://wis2box-api:80; } + location /wis2-downloader { + set $x_api_http_method $request_method; + auth_request /auth; + auth_request_set $auth_status $upstream_status; + rewrite ^/wis2-downloader/(.*) /$1 break; + proxy_pass http://wis2-downloader:5000; + } location /wis2box-webapp/ { auth_basic "Restricted Access"; auth_basic_user_file /etc/nginx/.htpasswd/webapp; diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 10ae797f..16d7d5c9 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -26,3 +26,8 @@ scrape_configs: - targets: ['cadvisor:8080'] labels: alias: 'cadvisor' +- job_name: wis2-downloader + metrics_path: /metrics + scheme: http + static_configs: + - targets: ['wis2-downloader:5000'] diff --git a/wis2-downloader/Dockerfile b/wis2-downloader/Dockerfile new file mode 100644 index 00000000..a5ee6fd1 --- /dev/null +++ b/wis2-downloader/Dockerfile @@ -0,0 +1,46 @@ +# Use an official Python runtime as a parent image +FROM python:3.9-slim + +EXPOSE 5000 + +# define config variables +ENV DOWNLOAD_DIR /app/downloads +ENV DOWNLOAD_RETENTION_PERIOD_HOURS 24 +ENV DOWNLOAD_BROKER_HOST "globalbroker.meteo.fr" +ENV DOWNLOAD_BROKER_PORT 443 +ENV DOWNLOAD_BROKER_USERNAME "everyone" +ENV DOWNLOAD_BROKER_PASSWORD "everyone" +ENV DOWNLOAD_BROKER_PROTOCOL "websockets" +ENV DOWNLOAD_WORKERS 8 +ENV FLASK_HOST "0.0.0.0" +ENV FLASK_PORT 5000 +ENV SAVE_LOGS false +ENV LOGS_DIR /app/logs + +# update pyopenssl and pin requests and urllib3 to avoid SSL error +RUN pip install pyopenssl --upgrade && pip install requests==2.26.0 urllib3==1.26.0 +# install cron and envsubst +RUN apt-get update && apt-get install -y cron gettext-base + +# copy all the code to the Docker image +COPY . /app + +# Set the working directory to /app +WORKDIR /app + +# install wis2-downloader +RUN pip install https://github.com/wmo-im/wis2-downloader/archive/main.zip + +# add wis2box.cron to crontab +COPY ./clean.cron /etc/cron.d/clean.cron + +# set permissions for the cron job and install it +RUN chmod 0644 /etc/cron.d/clean.cron && crontab /etc/cron.d/clean.cron + +# set permissions for the entrypoint script +RUN chmod +x /app/entrypoint.sh + +ENTRYPOINT [ "/app/entrypoint.sh" ] + +# Run wis2-downloader when the container launches +CMD ["wis2downloader","--config","/app/config.json"] diff --git a/wis2-downloader/clean.cron b/wis2-downloader/clean.cron new file mode 100644 index 00000000..1ae6e9c1 --- /dev/null +++ b/wis2-downloader/clean.cron @@ -0,0 +1 @@ +*/10 * * * * /usr/local/bin/python /app/clean.py > /proc/1/fd/1 2>/proc/1/fd/2 diff --git a/wis2-downloader/clean.py b/wis2-downloader/clean.py new file mode 100644 index 00000000..2fd2dc7e --- /dev/null +++ b/wis2-downloader/clean.py @@ -0,0 +1,39 @@ +import os +import time + +# get download dir from environment variable +download_dir = os.environ.get('DOWNLOAD_DIR', '/app/downloads') +# get retention period from environment variable +retention_period_hours = int(os.environ.get('RETENTION_PERIOD_HOURS', 24)) + + +def clean_directory(directory): + # get the current time + current_time = time.time() + + files_removed = 0 + directories_removed = 0 + # loop through the files in the directory, including subdirectories + for file in os.listdir(directory): + # get the full path of the file + file_path = os.path.join(directory, file) + # check if the path is a file or a directory + if os.path.isfile(file_path): + # get the time the file was last modified + file_time = os.path.getmtime(file_path) + # check if the file is older than the retention period + if current_time - file_time > retention_period_hours * 3600: + os.remove(file_path) + files_removed += 1 + elif os.path.isdir(file_path): + # recursively clean the directory + clean_directory(file_path) + # if the directory is empty, remove it + if not os.listdir(file_path): + os.rmdir(file_path) + directories_removed += 1 + print(f'CLEANER: removed {files_removed} old files and {directories_removed} empty directories') # noqa + + +# start cleaning from the download directory +clean_directory(download_dir) diff --git a/wis2-downloader/config.template b/wis2-downloader/config.template new file mode 100644 index 00000000..c5de79ac --- /dev/null +++ b/wis2-downloader/config.template @@ -0,0 +1,15 @@ +{ + "broker_url": "${DOWNLOAD_BROKER_HOST}", + "broker_port": $DOWNLOAD_BROKER_PORT, + "username": "${DOWNLOAD_BROKER_USERNAME}", + "password": "${DOWNLOAD_BROKER_PASSWORD}", + "protocol": "${DOWNLOAD_BROKER_PROTOCOL}", + "topics": {}, + "download_dir": "${DOWNLOAD_DIR}", + "retention_period_hours": "${DOWNLOAD_RETENTION_PERIOD_HOURS}", + "flask_host": "${FLASK_HOST}", + "flask_port": "${FLASK_PORT}", + "download_workers": $DOWNLOAD_WORKERS, + "save_logs": false, + "log_dir": "logs" +} diff --git a/wis2-downloader/entrypoint.sh b/wis2-downloader/entrypoint.sh new file mode 100644 index 00000000..ff9a19d1 --- /dev/null +++ b/wis2-downloader/entrypoint.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "START /entrypoint.sh" + +set -e + +# ensure cron is running +service cron start +service cron status + +echo "END /entrypoint.sh" + +# print the download_dir +echo "Download directory in container: $DOWNLOAD_DIR" +# print the retention period hours +echo "Retention period in hours: $RETENTION_PERIOD_HOURS" +# print the maximum MB allowed in the download directory +echo "Maximum MB allowed in the download directory: $MAX_MB_DOWNLOAD_DIR" + +# ensure DOWNLOAD_DIR exists +if [ ! -d $DOWNLOAD_DIR ]; then + echo "Creating download directory: $DOWNLOAD_DIR" + mkdir -p $DOWNLOAD_DIR +fi +envsubst < config.template > config.json + +# print the config +echo "Config:" +cat /app/config.json + +exec "$@" diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index c7a5d671..5d065f8b 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -91,6 +91,15 @@ else # Add the token wis2box auth add-token --path collections/stations -y fi +# repeat for wis2-downloader +is_restricted=$(wis2box auth is-restricted-path --path wis2-downloader) +if [ "$is_restricted" = "True" ]; then + echo "wis2-downloader is restricted" +else + echo "restricting wis2-downloader" + # Add the token + wis2box auth add-token --path wis2-downloader -y +fi echo "END /entrypoint.sh" exec "$@" diff --git a/wis2box-subscribe-download/Dockerfile b/wis2box-subscribe-download/Dockerfile deleted file mode 100644 index 6ce52a74..00000000 --- a/wis2box-subscribe-download/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -############################################################################### -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -############################################################################### - -FROM python:3.8 - -RUN pip3 install pywis-pubsub -# sync WIS2 notification schema -RUN pywis-pubsub schema sync - -COPY local.yml /app/ - -CMD ["pywis-pubsub", "subscribe", "--config", "/app/local.yml", "--download"] diff --git a/wis2box-subscribe-download/local.yml b/wis2box-subscribe-download/local.yml deleted file mode 100644 index 6034db2a..00000000 --- a/wis2box-subscribe-download/local.yml +++ /dev/null @@ -1,14 +0,0 @@ -broker: mqtts://host:port - -verify_data: true - -#validate_message: true - -subscribe_topics: - - '#' - -# storage: filesystem -storage: - type: fs - options: - path: /data/wis2box-subscribe-download