diff --git a/docker-compose.yml b/docker-compose.yml index 56579a65..33e4f09b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,10 +6,14 @@ services: environment: - POSTGRES_USER=${DB_USER:-objects} - POSTGRES_PASSWORD=${DB_PASSWORD:-objects} + networks: + - objects-dev redis: image: redis:7 command: ["redis-server", "--appendonly", "yes"] + networks: + - objects-dev web: image: maykinmedia/objects-api:latest @@ -27,6 +31,7 @@ services: CELERY_LOGLEVEL: DEBUG DISABLE_2FA: ${DISABLE_2FA:-yes} SUBPATH: ${SUBPATH} + LOG_LEVEL: INFO healthcheck: test: ["CMD", "python", "-c", "import requests; exit(requests.head('http://localhost:8000/admin/').status_code not in [200, 302])"] interval: 30s @@ -43,6 +48,13 @@ services: condition: service_completed_successfully volumes: &web_volumes - media:/app/media # Shared media volume to get access to saved OAS files + networks: + - objects-dev + labels: + client: dev + target: test + app: objects-api + service: web web-init: image: maykinmedia/objects-api:latest @@ -58,6 +70,46 @@ services: - redis volumes: - ./docker/setup_configuration:/app/setup_configuration + networks: + - objects-dev + + objecttypes-web: + image: maykinmedia/objecttypes-api:latest + environment: &app-env + DB_USER: objects + DB_PASSWORD: objects + DJANGO_SETTINGS_MODULE: objecttypes.conf.docker + SECRET_KEY: ${SECRET_KEY:-fgv=c0hz&tl*8*3m3893@m+1pstrvidc9e^5@fpspmg%cyf15d} + ALLOWED_HOSTS: '*' + CACHE_DEFAULT: redis:6379/0 + CACHE_AXES: redis:6379/0 + DISABLE_2FA: yes + SUBPATH: ${SUBPATH:-/} + volumes: + - ./docker/objecttypes/objecttypes.json:/app/fixtures/objecttypes.json + ports: + - 8001:8000 + depends_on: + objecttypes-web-init: + condition: service_completed_successfully + networks: + - objects-dev + + objecttypes-web-init: + image: maykinmedia/objecttypes-api:latest + environment: + <<: *app-env + # + # Django-setup-configuration + RUN_SETUP_CONFIG: ${RUN_SETUP_CONFIG:-true} + command: /setup_configuration.sh + volumes: + - ./docker/setup_configuration/objecttypes_data.yaml:/app/setup_configuration/data.yaml + depends_on: + - db + - redis + networks: + - objects-dev celery: image: maykinmedia/objects-api:latest @@ -74,6 +126,8 @@ services: web: condition: service_healthy volumes: *web_volumes + networks: + - objects-dev celery-flower: image: maykinmedia/objects-api:latest @@ -85,6 +139,12 @@ services: depends_on: - redis - celery + networks: + - objects-dev volumes: media: + +networks: + objects-dev: + name: objects-dev diff --git a/docker/docker-compose.observability.yaml b/docker/docker-compose.observability.yaml new file mode 100644 index 00000000..02f1d6d4 --- /dev/null +++ b/docker/docker-compose.observability.yaml @@ -0,0 +1,63 @@ +--- + +# Taken and adapted from https://grafana.com/docs/loki/latest/setup/install/docker/#install-with-docker-compose + +services: + loki: + image: grafana/loki:latest + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - objects-dev + + promtail: + image: grafana/promtail:latest + volumes: + # for service discovery & reading container logs - note that doing this in + # production is NOT recommended due to security concerns + - /var/run/docker.sock:/var/run/docker.sock:ro + # custom config file to scrape container logs + - ./observability/promtail/config.yml:/etc/promtail/config.yml + - promtail-logs:/var/log + command: -config.file=/etc/promtail/config.yml + networks: + - objects-dev + + grafana: + environment: + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_FEATURE_TOGGLES_ENABLE=alertingSimplifiedRouting,alertingQueryAndExpressionsStepMode + entrypoint: + - sh + - -euc + - | + mkdir -p /etc/grafana/provisioning/datasources + cat < /etc/grafana/provisioning/datasources/ds.yaml + apiVersion: 1 + datasources: + - name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false + EOF + /run.sh + image: grafana/grafana:latest + ports: + - "3000:3000" + networks: + - objects-dev + +volumes: + promtail-logs: + +networks: + objects-dev: + name: objects-dev diff --git a/docker/objecttypes/objecttypes.json b/docker/objecttypes/objecttypes.json new file mode 100644 index 00000000..8941df39 --- /dev/null +++ b/docker/objecttypes/objecttypes.json @@ -0,0 +1,38 @@ +[ + { + "model": "core.objecttype", + "pk": 1, + "fields": { + "uuid": "b427ef84-189d-43aa-9efd-7bb2c459e281", + "name": "Accepts everything", + "name_plural": "Accepts everything", + "description": "", + "data_classification": "open", + "maintainer_organization": "", + "maintainer_department": "", + "contact_person": "", + "contact_email": "", + "source": "", + "update_frequency": "unknown", + "provider_organization": "", + "documentation_url": "", + "labels": {}, + "created_at": "2025-05-16", + "modified_at": "2025-05-16", + "allow_geometry": true + } + }, + { + "model": "core.objectversion", + "pk": 1, + "fields": { + "object_type": 1, + "version": 1, + "created_at": "2025-05-16", + "modified_at": "2025-05-16", + "published_at": "2025-05-16", + "json_schema": true, + "status": "published" + } + } +] \ No newline at end of file diff --git a/docker/observability/README.md b/docker/observability/README.md new file mode 100644 index 00000000..f9f70ad0 --- /dev/null +++ b/docker/observability/README.md @@ -0,0 +1,50 @@ +# Observability stack + +The config files in this directory go together with `docker-compose.observability.yaml` and provide +a reference for a potential observability stack. + +**Disclaimer** + +The chosen vendors/technologies here merely serve as an example - it's simply a stack we're somewhat +comfortable with. Open Forms itself is vendor agnostic and the principles demonstrated apply to +competing vendors too. + +## Bringing up the services + +From the root of the repository: + +```bash +docker compose up -d +docker compose -f docker/docker-compose.observability.yml up +``` + +You can now navigate to: + +- http://localhost:3000 for Grafana +- http://localhost:3100/ready for Loki readiness +- http://localhost:3100/metrics for Loki metrics + +## Logging + +For log scraping, parsing and querying we've set up Promtail as scraper, Loki as storage and Grafana +as visualization tool. + +### Sample queries + +In the Grafana menu, navigate to "Explore" to create ad-hoc queries. + +**Web service logs** + +```logql +{job="docker", app="openforms"} | json | __error__ = "" +``` + +This ignores logs that cannot be parsed as JSON (such as container/server startup logs). + +**Logs for a single request** + +You can filter application logs based on a request ID: + +```logql +{job="docker", app="openforms"} | json | __error__ = "" | request_id=`1e9e1b9d-4d34-4657-99e4-88673d824724` +``` diff --git a/docker/observability/promtail/config.yml b/docker/observability/promtail/config.yml new file mode 100644 index 00000000..452245e8 --- /dev/null +++ b/docker/observability/promtail/config.yml @@ -0,0 +1,55 @@ +--- + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + # default config from the container, scraping itself + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log + # docker service discovery + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: + - __meta_docker_container_name + regex: /(.*) + target_label: container + - source_labels: + - __meta_docker_container_log_stream + target_label: logstream + - source_labels: + - __meta_docker_container_label_client + target_label: client + - source_labels: + - __meta_docker_container_label_app + target_label: app + - source_labels: + - __meta_docker_container_label_target + target_label: target + - source_labels: + - __meta_docker_container_label_service + target_label: service + - source_labels: + - __meta_docker_container_label_app + action: drop + regex: promtail + pipeline_stages: + - docker: {} + - static_labels: + job: docker + instance: docker.host.internal diff --git a/docker/setup_configuration/data.yaml b/docker/setup_configuration/data.yaml index 2842181c..8fb91c49 100644 --- a/docker/setup_configuration/data.yaml +++ b/docker/setup_configuration/data.yaml @@ -10,7 +10,7 @@ zgw_consumers: services: - identifier: objecttypes-api label: Objecttypes API - api_root: http://objecttypes.local/api/v1/ + api_root: http://objecttypes-web:8000/api/v2/ api_connection_check_path: objecttypes api_type: orc auth_type: api_key @@ -68,7 +68,14 @@ tokenauth: # additional permissions can be added like this: # - object_type: b427ef84-189d-43aa-9efd-7bb2c459e281 # mode: read_and_write - + - identifier: token-2 + token: QRjwfpD3nf0WpFOOHR37oMhcPPbWFKd4 + contact_person: Admin + email: admin@example.com + organization: Organization 2 + application: Application 2 + administration: Administration 2 + is_superuser: true oidc_db_config_enable: true oidc_db_config_admin_auth: diff --git a/docker/setup_configuration/objecttypes_data.yaml b/docker/setup_configuration/objecttypes_data.yaml new file mode 100644 index 00000000..9771d610 --- /dev/null +++ b/docker/setup_configuration/objecttypes_data.yaml @@ -0,0 +1,10 @@ +tokenauth_config_enable: true +tokenauth: + items: + - identifier: objects + token: b9f100590925b529664ed9d370f5f8da124b2c20 + contact_person: Person 1 + email: person-1@example.com + organization: Organization 1 + application: Application 1 + administration: Administration 1