From 97425565ce2adccfcaae16a357561a6cfb1ffbc9 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 3 May 2024 15:36:45 +0200 Subject: [PATCH 01/11] Bundle Postgres with Django server --- Dockerfile | 54 +++++++++++++++++++++++++----- backend/news/models.py | 9 +++-- dev.env | 14 -------- docker-compose.yml | 76 ++++++------------------------------------ run-dev.sh | 9 +++++ run-postgres.sh | 12 +++++++ run-preheating.sh | 18 ++++++++++ run-server.sh | 8 +++++ 8 files changed, 108 insertions(+), 92 deletions(-) delete mode 100644 dev.env create mode 100755 run-dev.sh create mode 100755 run-postgres.sh create mode 100755 run-preheating.sh create mode 100755 run-server.sh diff --git a/Dockerfile b/Dockerfile index 8e0ded0..814ffae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,59 @@ -FROM python:3.8 +FROM postgis/postgis:14-3.3 AS builder +# This docker image is based on the bullseye operating system +# See: https://github.com/postgis/docker-postgis/blob/master/14-3.3/Dockerfile -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 +# Install libraries needed for GeoDjango and PostGIS +# See https://docs.djangoproject.com/en/3.2/ref/contrib/gis/install/geolibs/ +RUN apt-get update && apt-get install -y \ + binutils \ + libproj-dev \ + gdal-bin # Install Postgres client to check liveness of the database -RUN apt-get update && apt-get install -y postgresql-client +RUN apt-get install -y postgresql-client + +# Install osm2pgsql to load the osm-data into the database +RUN apt-get install -y osm2pgsql + +# Install Python and a dependency for psycopg2 +RUN apt-get install -y python3 python3-pip libpq-dev # Install Poetry as the package manager for this application RUN pip install poetry WORKDIR /code -# Use the admin interface to check the health of the application -HEALTHCHECK --interval=10s --timeout=8s --start-period=20s --retries=10 \ - CMD curl --fail http://localhost:8000/admin || exit 1 - # Install Python dependencies separated from the # run script to enable Docker caching ADD pyproject.toml /code +# Install all dependencies RUN poetry install --no-interaction --no-ansi --no-dev -ADD . /code +# Install CURL for healthcheck +RUN apt-get update && apt-get install -y curl + +# Expose Django port, DO NOT EXPOSE THE DATABASE PORT! +EXPOSE 8000 + +COPY . /code + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +ENV POSTGRES_NAME=news-db +ENV POSTGRES_USER=news-user +ENV POSTGRES_PASSWORD=news-password +ENV POSTGRES_DB=news-db +ENV POSTGRES_HOST=localhost +ENV POSTGRES_PORT=5432 + +# Use this argument to invalidate the cache of subsequent steps. +ARG CACHE_DATE=1970-01-01 + +FROM builder AS production +ENV DJANGO_DEBUG_MODE=False +# Preheat our database, by running migrations and pre-loading data +RUN ./run-preheating.sh +HEALTHCHECK --interval=10s --timeout=8s --start-period=20s --retries=10 \ + CMD curl --fail http://localhost:8000/admin || exit 1 +CMD "./run-server.sh" diff --git a/backend/news/models.py b/backend/news/models.py index e36e0aa..b527f43 100644 --- a/backend/news/models.py +++ b/backend/news/models.py @@ -91,8 +91,11 @@ def send_notification_for_news_article(sender, instance, created, **kwargs): message = Message(notification=notification, topic=topic) # Send a message to the devices subscribed to the provided topic. - response = send(message) - # Response is a message ID string. - print('Successfully sent FCM message:', response) + if settings.FCM_PUSH_NOTIFICATION_ENVIRONMENT == 'dev': + print('[DEV] Sending message to topic:', topic) + else: + response = send(message) + # Response is a message ID string. + print('Successfully sent FCM message:', response) delete_app(app) diff --git a/dev.env b/dev.env deleted file mode 100644 index 6c7e4cd..0000000 --- a/dev.env +++ /dev/null @@ -1,14 +0,0 @@ -POSTGRES_NAME = news-db -POSTGRES_USER = news-user -POSTGRES_PASSWORD = news-password -POSTGRES_DB = news-db -POSTGRES_HOST = db -POSTGRES_PORT = 5432 -DJANGO_KEY = django-insecure-h4wvbex$t4k_adc#55mqy3d5x($(8qo5=-c_3x81vbrw*0u5pc -DJANGO_SUPERUSER_USERNAME=admin -DJANGO_SUPERUSER_EMAIL=admin@example.com -DJANGO_SUPERUSER_PASSWORD=secret -DEBUG = False -APP_URL = /production/news-service/ -FCM_PUSH_NOTIFICATION_ENVIRONMENT = dev -CSRF_TRUSTED_ORIGIN = http://127.0.0.1:20051 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c5ec5bd..4223ae4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,83 +3,27 @@ networks: name: production-network services: - db: - image: postgres - container_name: postgres - hostname: db - networks: - - production-network - env_file: - - dev.env + manager: + container_name: manager volumes: + - news_service_staticfiles:/code/backend/static/ + # Persist news articles that get created. - pg_conf:/etc/postgresql - pg_log:/var/log/postgresql - pg_data:/var/lib/postgresql/data - # Log all statements - # See: https://postgresqlco.nf/doc/en/param/log_statement/ - command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr"] - restart: unless-stopped - - backend: - container_name: backend - volumes: - - news_service_staticfiles:/code/backend/static/ - networks: - - production-network - build: . - env_file: - - dev.env - depends_on: - - db - command: - - /bin/bash - - -c - - | - ./wait-for-postgres.sh - poetry run python backend/manage.py migrate - poetry run python backend/manage.py collectstatic --no-input - poetry run python backend/manage.py createsuperuser --noinput - poetry run python backend/manage.py runserver 0.0.0.0:8000 - restart: unless-stopped - - nginx_inner: - image: nginx:1.21.6 - container_name: nginx_inner - hostname: nginx_inner - volumes: - - ./nginx_inner/conf.d:/etc/nginx/conf.d - - ./nginx_inner/proxy_params:/etc/nginx/proxy_params - # The static files for the news service - - news_service_staticfiles:/news-service/backend/static/ networks: - production-network - depends_on: - - db - - backend - restart: unless-stopped - - nginx_outer: - image: nginx:1.21.6 - container_name: nginx_outer - hostname: nginx_outer - volumes: - - ./nginx_outer/conf.d:/etc/nginx/conf.d - - ./nginx_outer/proxy_params:/etc/nginx/proxy_params - # The NGINX proxy service is the only service that listens to the outside world ports: - # This is important for the root NGINX proxy of the server. - - "20051:80" # HTTP - networks: - - production-network - depends_on: - - db - - backend - - nginx_inner + - "8000:8000" + build: . + environment: + - DJANGO_DEBUG_MODE=True + - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev + command: ./run-dev.sh restart: unless-stopped volumes: pg_data: pg_conf: pg_log: - news_service_staticfiles: diff --git a/run-dev.sh b/run-dev.sh new file mode 100755 index 0000000..a9de97b --- /dev/null +++ b/run-dev.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Run postgres in the background +./run-postgres.sh + +# Run the development server +cd backend +poetry run python manage.py migrate +poetry run python manage.py runserver 0.0.0.0:8000 \ No newline at end of file diff --git a/run-postgres.sh b/run-postgres.sh new file mode 100755 index 0000000..bc3e422 --- /dev/null +++ b/run-postgres.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Run postgres in the background +echo "Starting postgres..." +/usr/local/bin/docker-entrypoint.sh postgres \ + -c log_destination=stderr \ + -c max_parallel_workers_per_gather=4 \ + & +pid=$! +echo "Postgres started with pid $pid" + +./wait-for-postgres.sh \ No newline at end of file diff --git a/run-preheating.sh b/run-preheating.sh new file mode 100755 index 0000000..02f1698 --- /dev/null +++ b/run-preheating.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +echo "Preheating the docker image..." + +# Run postgres in the background +./run-postgres.sh + +# Check if previous command failed. If it did, exit +ret=$? +if [ $ret -ne 0 ]; then + echo "Failed to start postgres" + exit $ret +fi + +# Run the migration script +poetry run python backend/manage.py migrate + +echo "Preheating complete!" \ No newline at end of file diff --git a/run-server.sh b/run-server.sh new file mode 100755 index 0000000..2b9e4eb --- /dev/null +++ b/run-server.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Run postgres in the background +./run-postgres.sh + +# Run gunicorn +cd backend +poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8000 \ No newline at end of file From b4f7c49e46e172c48f4499ab9e0d92606f8b9b8d Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 3 May 2024 16:24:16 +0200 Subject: [PATCH 02/11] Small test setup with docker compose and traefik --- Dockerfile | 6 +++-- backend/backend/settings.py | 8 +++++- backend/backend/urls.py | 7 ++++- backend/backend/views.py | 46 ++++++++++++++++++++++++++++++++ docker-compose.yml | 53 +++++++++++++++++++++++++++++++------ 5 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 backend/backend/views.py diff --git a/Dockerfile b/Dockerfile index 814ffae..6922e02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,8 @@ ENV POSTGRES_DB=news-db ENV POSTGRES_HOST=localhost ENV POSTGRES_PORT=5432 +ENV HEALTHCHECK_TOKEN=healthcheck-token + # Use this argument to invalidate the cache of subsequent steps. ARG CACHE_DATE=1970-01-01 @@ -54,6 +56,6 @@ FROM builder AS production ENV DJANGO_DEBUG_MODE=False # Preheat our database, by running migrations and pre-loading data RUN ./run-preheating.sh -HEALTHCHECK --interval=10s --timeout=8s --start-period=20s --retries=10 \ - CMD curl --fail http://localhost:8000/admin || exit 1 +HEALTHCHECK --interval=60s --timeout=10s --retries=5 --start-period=10s \ + CMD curl --fail http://localhost:8000/healthcheck?token=healthcheck-token || exit 1 CMD "./run-server.sh" diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 7d190ad..94cf69e 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -19,6 +19,8 @@ ALLOWED_HOSTS = ['*'] +HEALTHCHECK_TOKEN = os.environ.get('HEALTHCHECK_TOKEN', 'healthcheck-token') + # One of the following: 'dev' / 'staging' / 'production' FCM_PUSH_NOTIFICATION_ENVIRONMENT = os.environ.get('FCM_PUSH_NOTIFICATION_ENVIRONMENT', 'dev') FCM_PUSH_NOTIFICATION_CONF = os.path.join(BASE_DIR.parent, "config/fcm-key.json") @@ -29,6 +31,9 @@ # Detect whether it's a test run or not. TESTING = sys.argv[1:2] == ['test'] +# Detect whether we run in worker or manager mode. +WORKER_MODE = 'True' in os.environ.get('WORKER_MODE', 'False') + # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get('DEBUG', 'True') == 'True' @@ -56,13 +61,14 @@ INSTALLED_APPS = [ 'news', - 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', ] +if not WORKER_MODE: + INSTALLED_APPS.append('django.contrib.admin') MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', diff --git a/backend/backend/urls.py b/backend/backend/urls.py index 07cd9dd..cdd59e8 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -17,7 +17,12 @@ from django.contrib import admin from django.urls import include, path +from backend.views import HealthcheckView, StatusView + urlpatterns = [ path('news/', include('news.urls')), - path(settings.ADMIN_URL, admin.site.urls), + path('status', StatusView.as_view(), name='status'), + path('healthcheck', HealthcheckView.as_view(), name='healthcheck'), ] +if not settings.WORKER_MODE: + urlpatterns.append(path(settings.ADMIN_URL, admin.site.urls)) diff --git a/backend/backend/views.py b/backend/backend/views.py new file mode 100644 index 0000000..342d86a --- /dev/null +++ b/backend/backend/views.py @@ -0,0 +1,46 @@ +from datetime import datetime + +from django.apps import apps +from django.conf import settings +from django.http import JsonResponse +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import View + + +class StatusView(View): + """ + View to get the status of the signal group selector. + """ + + def get(self, request, *args, **kwargs): + """ + Handle the GET request. + """ + return JsonResponse({'status': 'ok'}) + + +@method_decorator(csrf_exempt, name='dispatch') +class HealthcheckView(View): + """ + View to get the healthcheck of the signal group selector. + """ + + def get(self, request, *args, **kwargs): + """ + Handle the GET request. + """ + token = settings.HEALTHCHECK_TOKEN + if token and token != request.GET.get('token'): + return JsonResponse({'status': 'unauthorized'}, status=401) + + # Fetch all objects from all models. + # This will heat the cache and make sure + # the database is available. + now = datetime.now() + for model in apps.get_models(): + model.objects.all() + time = (datetime.now() - now).total_seconds() + print(f'OK: Healthcheck took {time} seconds') + + return JsonResponse({'status': 'ok', 'time': time}) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4223ae4..81321a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,29 +1,66 @@ +version: '3' + networks: - production-network: - name: production-network + test-network: + name: test-network services: manager: - container_name: manager volumes: - - news_service_staticfiles:/code/backend/static/ # Persist news articles that get created. - pg_conf:/etc/postgresql - pg_log:/var/log/postgresql - pg_data:/var/lib/postgresql/data networks: - - production-network - ports: - - "8000:8000" + - test-network build: . + volumes: + - ./:/code environment: - DJANGO_DEBUG_MODE=True - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev command: ./run-dev.sh restart: unless-stopped + worker: + hostname: worker{{.Task.Slot}}.local + networks: + - test-network + build: . + volumes: + - ./:/code + environment: + - DJANGO_DEBUG_MODE=True + - WORKER_MODE=True + command: ./run-dev.sh + restart: unless-stopped + labels: + - traefik.enable=true + - traefik.http.routers.worker.rule=PathPrefix(`/`) + - traefik.http.routers.worker.entryPoints=web + - traefik.http.services.worker.loadbalancer.server.port=8000 + deploy: + mode: replicated + endpoint_mode: dnsrr + replicas: 2 + + traefik: + image: traefik:v2.9 + hostname: traefik + networks: + - test-network + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "80:80" + - "8080:8080" + command: + - --providers.docker + - --providers.docker.exposedbydefault=false + - --providers.docker.network=test-network + - --entryPoints.web.address=:80 + volumes: pg_data: pg_conf: pg_log: - news_service_staticfiles: From 07838e2cf10f4e1558a5053e761cd2419005cda0 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 3 May 2024 16:35:55 +0200 Subject: [PATCH 03/11] Add communication between manager and worker (no logic yet) --- backend/backend/settings.py | 9 +++++++++ backend/news/models.py | 28 ++++++++++++++++++++++++++++ docker-compose.yml | 4 ++++ 3 files changed, 41 insertions(+) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 94cf69e..97531f0 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -33,6 +33,15 @@ # Detect whether we run in worker or manager mode. WORKER_MODE = 'True' in os.environ.get('WORKER_MODE', 'False') +if not WORKER_MODE: + # Needed to find the workers. + WORKER_HOST = os.environ.get('WORKER_HOST') + if not WORKER_HOST: + raise ValueError('WORKER_HOST is not set.') + WORKER_PORT = os.environ.get('WORKER_PORT', 8000) +else: + WORKER_HOST = None + WORKER_PORT = None # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get('DEBUG', 'True') == 'True' diff --git a/backend/news/models.py b/backend/news/models.py index b527f43..eeef792 100644 --- a/backend/news/models.py +++ b/backend/news/models.py @@ -1,7 +1,9 @@ import hashlib import json import os +import socket +import requests from django.conf import settings from django.db import models from django.db.models.signals import post_save @@ -55,6 +57,32 @@ class Meta: ordering = ['-pub_date'] +@receiver(post_save, sender=NewsArticle) +def sync_workers(sender, instance, created, **kwargs): + """ + Sync the new news article with the worker instances. + """ + # Lookup all workers using DNS. + if settings.WORKER_MODE: + return + + host = settings.WORKER_HOST + port = settings.WORKER_PORT + + worker_hosts = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + worker_ips = [worker_host[4][0] for worker_host in worker_hosts] + + # TODO: Implement the sync logic here. + print(f"Syncing news article with workers: {worker_ips}") + + # Fetch the status for now + for worker_ip in worker_ips: + print(f"Fetching status from worker: {worker_ip}") + url = f"http://{worker_ip}:{port}/status" + response = requests.get(url) + print(f"Status from worker {worker_ip}: {response.json()}") + + @receiver(post_save, sender=NewsArticle) def send_notification_for_news_article(sender, instance, created, **kwargs): """ diff --git a/docker-compose.yml b/docker-compose.yml index 81321a3..300ab68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,10 @@ services: environment: - DJANGO_DEBUG_MODE=True - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev + - WORKER_HOST=worker + - WORKER_PORT=8000 + ports: + - "8000:8000" command: ./run-dev.sh restart: unless-stopped From 886059ce599d09ae0b63734659f8e3cd138dc209 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 6 May 2024 09:19:11 +0200 Subject: [PATCH 04/11] Implement sync between manager and worker --- Dockerfile | 3 ++ backend/backend/settings.py | 37 +++++++++---------- backend/backend/urls.py | 3 ++ backend/news/models.py | 37 +++++++++++++++---- backend/sync/__init__.py | 0 backend/sync/apps.py | 6 ++++ backend/sync/urls.py | 9 +++++ backend/sync/views.py | 71 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 3 +- run-dev.sh | 5 ++- run-server.sh | 5 ++- 11 files changed, 148 insertions(+), 31 deletions(-) create mode 100644 backend/sync/__init__.py create mode 100644 backend/sync/apps.py create mode 100644 backend/sync/urls.py create mode 100644 backend/sync/views.py diff --git a/Dockerfile b/Dockerfile index 6922e02..69d65e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,9 @@ ENV HEALTHCHECK_TOKEN=healthcheck-token # Use this argument to invalidate the cache of subsequent steps. ARG CACHE_DATE=1970-01-01 +# Make the postgres persistance dirs writable for every user +RUN chmod -R 777 /var/lib/postgresql/data + FROM builder AS production ENV DJANGO_DEBUG_MODE=False # Preheat our database, by running migrations and pre-loading data diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 97531f0..56ced42 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -38,14 +38,18 @@ WORKER_HOST = os.environ.get('WORKER_HOST') if not WORKER_HOST: raise ValueError('WORKER_HOST is not set.') - WORKER_PORT = os.environ.get('WORKER_PORT', 8000) else: WORKER_HOST = None - WORKER_PORT = None # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get('DEBUG', 'True') == 'True' +SYNC_PORT = os.environ.get('SYNC_PORT', 8001) +SYNC_EXPOSED = 'True' in os.environ.get('SYNC_EXPOSED', 'False') +SYNC_KEY = os.environ.get('SYNC_KEY') +if not SYNC_KEY: + raise ValueError('SYNC_KEY is not set.') + # The news service is deployed behind reverse NGINX proxies. # Therefore, we set the admin url here so that it redirects to the correct browser path. # By default, the admin site will be accessible under admin/ @@ -112,26 +116,17 @@ # Database # https://docs.djangoproject.com/en/4.0/ref/settings/#databases - -if DEBUG: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', - } - } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - - 'NAME': os.environ.get('POSTGRES_NAME'), - 'USER': os.environ.get('POSTGRES_USER'), - 'PASSWORD': os.environ.get('POSTGRES_PASSWORD'), - 'HOST': os.environ.get('POSTGRES_HOST'), - 'PORT': os.environ.get('POSTGRES_PORT'), - } +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + + 'NAME': os.environ.get('POSTGRES_NAME'), + 'USER': os.environ.get('POSTGRES_USER'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD'), + 'HOST': os.environ.get('POSTGRES_HOST'), + 'PORT': os.environ.get('POSTGRES_PORT'), } +} # Password validation diff --git a/backend/backend/urls.py b/backend/backend/urls.py index cdd59e8..bbab9a4 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -26,3 +26,6 @@ ] if not settings.WORKER_MODE: urlpatterns.append(path(settings.ADMIN_URL, admin.site.urls)) + +if settings.SYNC_EXPOSED: + urlpatterns.append(path('sync/', include('sync.urls'))) diff --git a/backend/news/models.py b/backend/news/models.py index eeef792..d54ace5 100644 --- a/backend/news/models.py +++ b/backend/news/models.py @@ -67,20 +67,43 @@ def sync_workers(sender, instance, created, **kwargs): return host = settings.WORKER_HOST - port = settings.WORKER_PORT + port = settings.SYNC_PORT worker_hosts = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) worker_ips = [worker_host[4][0] for worker_host in worker_hosts] - # TODO: Implement the sync logic here. - print(f"Syncing news article with workers: {worker_ips}") + # Write a json that contains all news articles and categories. + data = { + 'key': settings.SYNC_KEY, + 'categories': [ + { + 'title': category.title, + } for category in Category.objects.all() + ], + 'articles': [ + { + 'text': article.text, + 'title': article.title, + 'pubDate': article.pub_date.isoformat(), + 'category': article.category.title if article.category else None, + } for article in NewsArticle.objects.all() + ] + } # Fetch the status for now for worker_ip in worker_ips: - print(f"Fetching status from worker: {worker_ip}") - url = f"http://{worker_ip}:{port}/status" - response = requests.get(url) - print(f"Status from worker {worker_ip}: {response.json()}") + print(f"Syncing with worker: {worker_ip}") + url = f"http://{worker_ip}:{port}/sync/sync" + response = requests.post(url, json=data) + # Parse the response as json + if response.status_code != 200: + print(f"Failed to sync with worker {worker_ip}: status {response.status_code}") + raise Exception(f"Failed to sync with worker {worker_ip}: status {response.status_code}") + status = json.loads(response.text).get('status') + if status != 'ok': + print(f"Failed to sync with worker {worker_ip}: {status}") + raise Exception(f"Failed to sync with worker {worker_ip}: {status}") + print(f"Synced with worker {worker_ip}: {status}") @receiver(post_save, sender=NewsArticle) diff --git a/backend/sync/__init__.py b/backend/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/sync/apps.py b/backend/sync/apps.py new file mode 100644 index 0000000..64d221b --- /dev/null +++ b/backend/sync/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class SyncAppConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'sync' diff --git a/backend/sync/urls.py b/backend/sync/urls.py new file mode 100644 index 0000000..ba2f7ec --- /dev/null +++ b/backend/sync/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from . import views + +app_name = 'sync' + +urlpatterns = [ + path("sync", views.SyncResource.as_view(), name="sync"), +] diff --git a/backend/sync/views.py b/backend/sync/views.py new file mode 100644 index 0000000..26b06bb --- /dev/null +++ b/backend/sync/views.py @@ -0,0 +1,71 @@ +import json + +from django.conf import settings +from django.db.transaction import atomic +from django.http import HttpResponseBadRequest, JsonResponse +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import View +from news.models import Category, NewsArticle + + +@method_decorator(csrf_exempt, name='dispatch') +class SyncResource(View): + def post(self, request): + # Check that the sync key is correct. + try: + body = json.loads(request.body) + except json.JSONDecodeError: + print("Invalid JSON.") + return HttpResponseBadRequest(json.dumps({"error": "Invalid JSON."})) + + sync_key = body.get("key") + if sync_key != settings.SYNC_KEY: + print(f"Invalid key: {sync_key}") + return HttpResponseBadRequest(json.dumps({"error": "Invalid key."})) + + if not isinstance(body.get("categories"), list) or not isinstance(body.get("articles"), list): + print(f"Invalid data format: {body}") + return HttpResponseBadRequest(json.dumps({"error": "Invalid data format."})) + + with atomic(): + # Clear the database + Category.objects.all().delete() + NewsArticle.objects.all().delete() + + # Load the categories contained in the response. + categories = body.get("categories") + for category in categories: + title = category.get("title") + try: + _, created = Category.objects.get_or_create(title=title) + if created: + print(f"Created category: {title}") + except Exception as err: + print(f"Error during sync: {err}") + return JsonResponse({"status": "error"}) + + # Load the articles contained in the response. + articles = body.get("articles") + for article in articles: + category_title = article.get("category") + article_title = article.get("title") + if category_title: + category = Category.objects.get(title=category_title) + else: + category = None + try: + _, created = NewsArticle.objects.get_or_create( + text=article.get("text"), + title=article_title, + pub_date=timezone.datetime.fromisoformat(article.get("pubDate")), + category=category, + ) + except Exception as err: + print(f"Error during sync: {err}") + return JsonResponse({"status": "error"}) + if created: + print(f"Created article: {article_title}") + + return JsonResponse({"status": "ok"}) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 300ab68..36f3ddf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: - DJANGO_DEBUG_MODE=True - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev - WORKER_HOST=worker - - WORKER_PORT=8000 + - SYNC_KEY=secret ports: - "8000:8000" command: ./run-dev.sh @@ -36,6 +36,7 @@ services: environment: - DJANGO_DEBUG_MODE=True - WORKER_MODE=True + - SYNC_KEY=secret command: ./run-dev.sh restart: unless-stopped labels: diff --git a/run-dev.sh b/run-dev.sh index a9de97b..2a76b60 100755 --- a/run-dev.sh +++ b/run-dev.sh @@ -6,4 +6,7 @@ # Run the development server cd backend poetry run python manage.py migrate -poetry run python manage.py runserver 0.0.0.0:8000 \ No newline at end of file + +(SYNC_EXPOSED="True" poetry run python manage.py runserver 0.0.0.0:8001; [ "$?" -lt 2 ] && kill "$$") & +(SYNC_EXPOSED="False" poetry run python manage.py runserver 0.0.0.0:8000; [ "$?" -lt 2 ] && kill "$$") & +wait \ No newline at end of file diff --git a/run-server.sh b/run-server.sh index 2b9e4eb..5807008 100755 --- a/run-server.sh +++ b/run-server.sh @@ -5,4 +5,7 @@ # Run gunicorn cd backend -poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8000 \ No newline at end of file + +(SYNC_EXPOSED="True" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8001; [ "$?" -lt 2 ] && kill "$$") & +(SYNC_EXPOSED="False" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8000; [ "$?" -lt 2 ] && kill "$$") & +wait \ No newline at end of file From 4a6a334876016145ea1fadd72ad3dd490d0ad40e Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 6 May 2024 09:47:38 +0200 Subject: [PATCH 05/11] Sync worker on first startup --- backend/backend/settings.py | 1 + backend/news/models.py | 77 +++++++++++++++++++----- backend/sync/management/commands/sync.py | 36 +++++++++++ backend/sync/views.py | 55 ++++------------- docker-compose.yml | 1 + run-dev.sh | 8 +++ 6 files changed, 121 insertions(+), 57 deletions(-) create mode 100644 backend/sync/management/commands/sync.py diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 56ced42..4e7f61a 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -73,6 +73,7 @@ INSTALLED_APPS = [ 'news', + 'sync', 'django.contrib.auth', 'django.contrib.contenttypes', diff --git a/backend/news/models.py b/backend/news/models.py index d54ace5..e315468 100644 --- a/backend/news/models.py +++ b/backend/news/models.py @@ -7,6 +7,7 @@ from django.conf import settings from django.db import models from django.db.models.signals import post_save +from django.db.transaction import atomic from django.dispatch import receiver from django.utils import timezone from firebase_admin import delete_app, initialize_app @@ -57,21 +58,48 @@ class Meta: ordering = ['-pub_date'] -@receiver(post_save, sender=NewsArticle) -def sync_workers(sender, instance, created, **kwargs): - """ - Sync the new news article with the worker instances. - """ - # Lookup all workers using DNS. - if settings.WORKER_MODE: - return - - host = settings.WORKER_HOST - port = settings.SYNC_PORT - - worker_hosts = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) - worker_ips = [worker_host[4][0] for worker_host in worker_hosts] - +def sync_from_content(data): + with atomic(): + # Clear the database + Category.objects.all().delete() + NewsArticle.objects.all().delete() + + # Load the categories contained in the response. + categories = data.get("categories") + for category in categories: + title = category.get("title") + try: + _, created = Category.objects.get_or_create(title=title) + if created: + print(f"Created category: {title}") + except Exception as err: + print(f"Error during sync: {err}") + raise err + + # Load the articles contained in the response. + articles = data.get("articles") + for article in articles: + category_title = article.get("category") + article_title = article.get("title") + if category_title: + category = Category.objects.get(title=category_title) + else: + category = None + try: + _, created = NewsArticle.objects.get_or_create( + text=article.get("text"), + title=article_title, + pub_date=timezone.datetime.fromisoformat(article.get("pubDate")), + category=category, + ) + except Exception as err: + print(f"Error during sync: {err}") + raise err + if created: + print(f"Created article: {article_title}") + + +def get_sync_content(): # Write a json that contains all news articles and categories. data = { 'key': settings.SYNC_KEY, @@ -89,6 +117,25 @@ def sync_workers(sender, instance, created, **kwargs): } for article in NewsArticle.objects.all() ] } + return data + + +@receiver(post_save, sender=NewsArticle) +def sync_workers(sender, instance, created, **kwargs): + """ + Sync the new news article with the worker instances. + """ + # Lookup all workers using DNS. + if settings.WORKER_MODE: + return + + host = settings.WORKER_HOST + port = settings.SYNC_PORT + + worker_hosts = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + worker_ips = [worker_host[4][0] for worker_host in worker_hosts] + + data = get_sync_content() # Fetch the status for now for worker_ip in worker_ips: diff --git a/backend/sync/management/commands/sync.py b/backend/sync/management/commands/sync.py new file mode 100644 index 0000000..bf6ea14 --- /dev/null +++ b/backend/sync/management/commands/sync.py @@ -0,0 +1,36 @@ +import json + +import requests +from django.conf import settings +from django.core.management.base import BaseCommand +from news.models import sync_from_content + + +class Command(BaseCommand): + """ + Sync from the manager. + """ + + def add_arguments(self, parser): + parser.add_argument("--host", type=str, help="The host to sync from.") + parser.add_argument("--port", type=int, help="The port to sync from.") + + def handle(self, *args, **options): + if not options["host"]: + raise ValueError("Missing required argument: --host") + if not options["port"]: + raise ValueError("Missing required argument: --port") + + host = options["host"] + port = options["port"] + + # Get the data from the manager. + key = settings.SYNC_KEY + response = requests.get(f"http://{host}:{port}/sync/sync?key={key}") + if response.status_code != 200: + print(f"Error: {response.text}") + raise Exception(f"Failed to sync with manager: status {response.status_code}") + + data = json.loads(response.text) + sync_from_content(data) + \ No newline at end of file diff --git a/backend/sync/views.py b/backend/sync/views.py index 26b06bb..2f7d80c 100644 --- a/backend/sync/views.py +++ b/backend/sync/views.py @@ -1,17 +1,22 @@ import json from django.conf import settings -from django.db.transaction import atomic from django.http import HttpResponseBadRequest, JsonResponse -from django.utils import timezone from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt from django.views.generic import View -from news.models import Category, NewsArticle +from news.models import get_sync_content, sync_from_content @method_decorator(csrf_exempt, name='dispatch') class SyncResource(View): + def get(self, request): + sync_key = request.GET.get("key") + if sync_key != settings.SYNC_KEY: + print(f"Invalid key: {sync_key}") + return HttpResponseBadRequest(json.dumps({"error": "Invalid key."})) + return JsonResponse(get_sync_content()) + def post(self, request): # Check that the sync key is correct. try: @@ -29,43 +34,9 @@ def post(self, request): print(f"Invalid data format: {body}") return HttpResponseBadRequest(json.dumps({"error": "Invalid data format."})) - with atomic(): - # Clear the database - Category.objects.all().delete() - NewsArticle.objects.all().delete() - - # Load the categories contained in the response. - categories = body.get("categories") - for category in categories: - title = category.get("title") - try: - _, created = Category.objects.get_or_create(title=title) - if created: - print(f"Created category: {title}") - except Exception as err: - print(f"Error during sync: {err}") - return JsonResponse({"status": "error"}) - - # Load the articles contained in the response. - articles = body.get("articles") - for article in articles: - category_title = article.get("category") - article_title = article.get("title") - if category_title: - category = Category.objects.get(title=category_title) - else: - category = None - try: - _, created = NewsArticle.objects.get_or_create( - text=article.get("text"), - title=article_title, - pub_date=timezone.datetime.fromisoformat(article.get("pubDate")), - category=category, - ) - except Exception as err: - print(f"Error during sync: {err}") - return JsonResponse({"status": "error"}) - if created: - print(f"Created article: {article_title}") - + try: + sync_from_content(body) + except Exception as err: + print(f"Error during sync: {err}") + return HttpResponseBadRequest(json.dumps({"error": "Error during sync."})) return JsonResponse({"status": "ok"}) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 36f3ddf..6a0cff1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,7 @@ services: environment: - DJANGO_DEBUG_MODE=True - WORKER_MODE=True + - MANAGER_HOST=manager - SYNC_KEY=secret command: ./run-dev.sh restart: unless-stopped diff --git a/run-dev.sh b/run-dev.sh index 2a76b60..1e5ff9a 100755 --- a/run-dev.sh +++ b/run-dev.sh @@ -7,6 +7,14 @@ cd backend poetry run python manage.py migrate +# If WORKER_MODE is "True", we sync with the manager. +if [ "$WORKER_MODE" = "True" ]; then + until poetry run python manage.py sync --host "$MANAGER_HOST" --port 8001; do + echo "Manager is unavailable - sleeping" + sleep 1 + done +fi + (SYNC_EXPOSED="True" poetry run python manage.py runserver 0.0.0.0:8001; [ "$?" -lt 2 ] && kill "$$") & (SYNC_EXPOSED="False" poetry run python manage.py runserver 0.0.0.0:8000; [ "$?" -lt 2 ] && kill "$$") & wait \ No newline at end of file From 77a11cf402550dd7eb0d8a294c1cd348303d3280 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 6 May 2024 10:05:53 +0200 Subject: [PATCH 06/11] Fix unit test --- backend/backend/settings.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 4e7f61a..243a93d 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -36,8 +36,6 @@ if not WORKER_MODE: # Needed to find the workers. WORKER_HOST = os.environ.get('WORKER_HOST') - if not WORKER_HOST: - raise ValueError('WORKER_HOST is not set.') else: WORKER_HOST = None @@ -47,8 +45,6 @@ SYNC_PORT = os.environ.get('SYNC_PORT', 8001) SYNC_EXPOSED = 'True' in os.environ.get('SYNC_EXPOSED', 'False') SYNC_KEY = os.environ.get('SYNC_KEY') -if not SYNC_KEY: - raise ValueError('SYNC_KEY is not set.') # The news service is deployed behind reverse NGINX proxies. # Therefore, we set the admin url here so that it redirects to the correct browser path. From e78b0de355594b255939ac3df7938ca02acd6a3d Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 6 May 2024 10:07:30 +0200 Subject: [PATCH 07/11] Use sqlite during tests --- backend/backend/settings.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 243a93d..cc1fb06 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -125,6 +125,12 @@ } } +if TESTING: + DATABASES['default'] = { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } + # Password validation # https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators From 9b2f444eaa495c670dfa172453060eb9741a2b39 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 6 May 2024 10:09:14 +0200 Subject: [PATCH 08/11] Don't run post save hook in tests --- backend/news/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/news/models.py b/backend/news/models.py index e315468..a048c0b 100644 --- a/backend/news/models.py +++ b/backend/news/models.py @@ -128,6 +128,8 @@ def sync_workers(sender, instance, created, **kwargs): # Lookup all workers using DNS. if settings.WORKER_MODE: return + if settings.TESTING: + return host = settings.WORKER_HOST port = settings.SYNC_PORT From 3b69f6a11191b85f0916546db5241bc4e15d8055 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 10 May 2024 10:31:20 +0200 Subject: [PATCH 09/11] Remove outer/inner nginx setup --- nginx_inner/conf.d/default.conf | 43 --------------------------------- nginx_inner/proxy_params | 5 ---- nginx_outer/conf.d/default.conf | 38 ----------------------------- nginx_outer/proxy_params | 5 ---- 4 files changed, 91 deletions(-) delete mode 100644 nginx_inner/conf.d/default.conf delete mode 100644 nginx_inner/proxy_params delete mode 100644 nginx_outer/conf.d/default.conf delete mode 100644 nginx_outer/proxy_params diff --git a/nginx_inner/conf.d/default.conf b/nginx_inner/conf.d/default.conf deleted file mode 100644 index 122d37b..0000000 --- a/nginx_inner/conf.d/default.conf +++ /dev/null @@ -1,43 +0,0 @@ -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -upstream backend { - server backend:8000; -} - -server { - # Enable gzip compression and decompression - # See: https://docs.nginx.com/nginx/admin-guide/web-server/compression/ - gzip on; - gunzip on; - - listen 81 default_server; - # NOTE: Enable HTTPS only if the production environment supports it. - # listen [::]:80 default_server; - server_name localhost; - - add_header 'Access-Control-Allow-Origin' '$http_origin' always; - add_header 'Access-Control-Allow-Credentials' 'true' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - add_header 'Access-Control-Max-Age' 1728000; - - location /news-service/static/ { - autoindex on; - alias /news-service/backend/static/; - } - - location /news-service/ { - proxy_pass http://backend/; - - include /etc/nginx/proxy_params; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/nginx_inner/proxy_params b/nginx_inner/proxy_params deleted file mode 100644 index ee36826..0000000 --- a/nginx_inner/proxy_params +++ /dev/null @@ -1,5 +0,0 @@ -proxy_redirect off; -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Host $server_name; \ No newline at end of file diff --git a/nginx_outer/conf.d/default.conf b/nginx_outer/conf.d/default.conf deleted file mode 100644 index 5edeb00..0000000 --- a/nginx_outer/conf.d/default.conf +++ /dev/null @@ -1,38 +0,0 @@ -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -upstream nginx_inner { - server nginx_inner:81; -} - -server { - # Enable gzip compression and decompression - # See: https://docs.nginx.com/nginx/admin-guide/web-server/compression/ - gzip on; - gunzip on; - - listen 80 default_server; - # NOTE: Enable HTTPS only if the production environment supports it. - # listen [::]:80 default_server; - server_name localhost; - - add_header 'Access-Control-Allow-Origin' '$http_origin' always; - add_header 'Access-Control-Allow-Credentials' 'true' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST'; - add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; - add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; - add_header 'Access-Control-Max-Age' 1728000; - - location /production/ { - proxy_pass http://nginx_inner/; - - include /etc/nginx/proxy_params; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} diff --git a/nginx_outer/proxy_params b/nginx_outer/proxy_params deleted file mode 100644 index ee36826..0000000 --- a/nginx_outer/proxy_params +++ /dev/null @@ -1,5 +0,0 @@ -proxy_redirect off; -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Host $server_name; \ No newline at end of file From bdccd3d1aa1c9e431fee963fd1bc1159d16482d4 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 13 May 2024 12:57:08 +0200 Subject: [PATCH 10/11] Fix duplicate key in compose --- docker-compose.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6a0cff1..6150b78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,16 +6,15 @@ networks: services: manager: - volumes: - # Persist news articles that get created. - - pg_conf:/etc/postgresql - - pg_log:/var/log/postgresql - - pg_data:/var/lib/postgresql/data networks: - test-network build: . volumes: - ./:/code + # Persist news articles that get created. + - pg_conf:/etc/postgresql + - pg_log:/var/log/postgresql + - pg_data:/var/lib/postgresql/data environment: - DJANGO_DEBUG_MODE=True - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev From 4b0d57b642441458b33df9b5ff463bc3149eec61 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 13 May 2024 13:11:40 +0200 Subject: [PATCH 11/11] Polishing --- README.md | 2 +- docker-compose.yml | 1 + run-dev.sh | 39 +++++++++++++++++++++++++++++++++++---- run-server.sh | 42 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9948427..afe7274 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # News Service A microservice to create and publish news to the PrioBike app. -## REST-API Endpoints +## WORKER REST-API Endpoints - ```api/news```: Get all news articles - optional query params: - ```from```: Specifies the date(time) from which on new news articles shoud be returned.released on or before the ```from``` date(time). diff --git a/docker-compose.yml b/docker-compose.yml index 6150b78..f73dbdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - pg_log:/var/log/postgresql - pg_data:/var/lib/postgresql/data environment: + - DJANGO_SUPERUSER_PASSWORD=secret - DJANGO_DEBUG_MODE=True - FCM_PUSH_NOTIFICATION_ENVIRONMENT=dev - WORKER_HOST=worker diff --git a/run-dev.sh b/run-dev.sh index 1e5ff9a..2af356e 100755 --- a/run-dev.sh +++ b/run-dev.sh @@ -1,20 +1,51 @@ #!/bin/bash +# Based on: https://gist.github.com/mjambon/79adfc5cf6b11252e78b75df50793f24 + # Run postgres in the background ./run-postgres.sh -# Run the development server cd backend + +# Run the development server poetry run python manage.py migrate +pids=() + # If WORKER_MODE is "True", we sync with the manager. if [ "$WORKER_MODE" = "True" ]; then until poetry run python manage.py sync --host "$MANAGER_HOST" --port 8001; do echo "Manager is unavailable - sleeping" sleep 1 done +else + # Create a superuser for the manager + poetry run python manage.py createsuperuser \ + --noinput \ + --username admin \ + --email admin@example.com fi -(SYNC_EXPOSED="True" poetry run python manage.py runserver 0.0.0.0:8001; [ "$?" -lt 2 ] && kill "$$") & -(SYNC_EXPOSED="False" poetry run python manage.py runserver 0.0.0.0:8000; [ "$?" -lt 2 ] && kill "$$") & -wait \ No newline at end of file +SYNC_EXPOSED="True" poetry run python manage.py runserver 0.0.0.0:8001 & +pids+=($!) + +SYNC_EXPOSED="False" poetry run python manage.py runserver 0.0.0.0:8000 & +pids+=($!) + +# 'set -e' tells the shell to exit if any of the foreground command fails, +# i.e. exits with a non-zero status. +set -eu + +# Wait for each specific process to terminate. +# Instead of this loop, a single call to 'wait' would wait for all the jobs +# to terminate, but it would not give us their exit status. +# +for pid in "${pids[@]}"; do + # + # Waiting on a specific PID makes the wait command return with the exit + # status of that process. Because of the 'set -e' setting, any exit status + # other than zero causes the current shell to terminate with that exit + # status as well. + # + wait "$pid" +done diff --git a/run-server.sh b/run-server.sh index 5807008..5c7d911 100755 --- a/run-server.sh +++ b/run-server.sh @@ -1,11 +1,45 @@ #!/bin/bash +# Based on: https://gist.github.com/mjambon/79adfc5cf6b11252e78b75df50793f24 + # Run postgres in the background ./run-postgres.sh -# Run gunicorn cd backend -(SYNC_EXPOSED="True" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8001; [ "$?" -lt 2 ] && kill "$$") & -(SYNC_EXPOSED="False" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8000; [ "$?" -lt 2 ] && kill "$$") & -wait \ No newline at end of file +# Run gunicorn +poetry run python manage.py migrate + +pids=() + +# If WORKER_MODE is "True", we sync with the manager. +if [ "$WORKER_MODE" = "True" ]; then + until poetry run python manage.py sync --host "$MANAGER_HOST" --port 8001; do + echo "Manager is unavailable - sleeping" + sleep 1 + done +fi + +SYNC_EXPOSED="True" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8001 & +pids+=($!) + +SYNC_EXPOSED="False" poetry run gunicorn backend.wsgi:application --workers 4 --bind 0.0.0.0:8000 & +pids+=($!) + +# 'set -e' tells the shell to exit if any of the foreground command fails, +# i.e. exits with a non-zero status. +set -eu + +# Wait for each specific process to terminate. +# Instead of this loop, a single call to 'wait' would wait for all the jobs +# to terminate, but it would not give us their exit status. +# +for pid in "${pids[@]}"; do + # + # Waiting on a specific PID makes the wait command return with the exit + # status of that process. Because of the 'set -e' setting, any exit status + # other than zero causes the current shell to terminate with that exit + # status as well. + # + wait "$pid" +done