Load data from overpass, add preheating/healthcheck etc., add workflows

.github/workflows/nightly.yaml

@@ -0,0 +1,83 @@
+name: Nightly
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yaml
+    with: 
+      branch: ${{ github.ref }}
+
+  publish-hamburg-main:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-hamburg
+      branch: main
+      location: Hamburg
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-dresden-main:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-dresden
+      branch: main
+      location: Dresden
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-hamburg-stable:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-hamburg
+      branch: stable
+      location: Hamburg
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-dresden-stable:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-dresden
+      branch: stable
+      location: Dresden
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-hamburg-release:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-hamburg
+      branch: release
+      location: Hamburg
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-dresden-release:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-dresden
+      branch: release
+      location: Dresden
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}

.github/workflows/on-pr.yaml

@@ -0,0 +1,14 @@
+name: Build
+
+on:
+  pull_request:
+    branches:
+      - main
+      - stable
+      - release
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yaml
+    with: 
+      branch: ${{ github.ref }}

.github/workflows/on-push.yaml

@@ -0,0 +1,50 @@
+name: Build
+
+on:
+  push:
+    branches:
+      - main
+      - stable
+      - release
+
+jobs:
+  test:
+    uses: ./.github/workflows/test.yaml
+    with: 
+      branch: ${{ github.ref }}
+
+  publish-hamburg-stable:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-hamburg
+      branch: stable
+      location: Hamburg
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-hamburg-release:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-hamburg
+      branch: release
+      location: Hamburg
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
+
+  publish-dresden:
+    needs: test
+    uses: ./.github/workflows/publish.yaml
+    with:
+      registry: bikenow.vkw.tu-dresden.de
+      image: ${{ github.repository }}-dresden
+      branch: ${{ github.ref_name }}
+      location: Dresden
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
+      NEXUS_DOCKER_PUB_PASSWORD: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}

.github/workflows/publish.yaml

@@ -1,21 +1,35 @@
-name: Build
+name: Create and publish Docker image
 
 on:
-  push:
-    branches:
-      - main
-      - stable
-      - release
-    tags:
-      - v*
-
-env:
-  REGISTRY: bikenow.vkw.tu-dresden.de
-  IMAGE_NAME: ${{ github.repository }}
+  workflow_call:
+    inputs:
+      registry:
+        description: 'The Docker registry'
+        required: true
+        type: string
+      image:
+        description: 'The Docker image name'
+        required: true
+        type: string
+      branch:
+        description: 'The branch to build'
+        required: true
+        type: string
+      location:
+        description: 'The location for which POIs should be fetched from overpass turbo.'
+        required: true
+        type: string
+    secrets:
+      NEXUS_DOCKER_PUB_USERNAME:
+        description: 'Username for the Nexus Docker registry'
+        required: true
+      NEXUS_DOCKER_PUB_PASSWORD:
+        description: 'Password for the Nexus Docker registry'
+        required: true
 
 jobs:
   publish:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     container: docker:latest
 
     permissions:
@@ -25,11 +39,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.branch }}
 
       - name: Log in to the container registry
         uses: docker/login-action@v2
         with:
-          registry: ${{ env.REGISTRY }}
+          registry: ${{ inputs.registry }}
           username: ${{ secrets.NEXUS_DOCKER_PUB_USERNAME }}
           password: ${{ secrets.NEXUS_DOCKER_PUB_PASSWORD }}
 
@@ -38,12 +54,18 @@ jobs:
         # See: https://github.com/docker/metadata-action
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ inputs.registry }}/${{ inputs.image }}
+          tags: |
+            type=raw,value=${{ inputs.branch }}
 
       - name: Build and push Docker image
         uses: docker/build-push-action@v4
         with:
-          context: .
+          context: backend
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          target: production # Preheat database to be immediately ready for production
+          build-args: |
+            LOCATION=${{ inputs.location }}
+            CACHE_DATE=${{ github.event.repository.updated_at }}-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/test.yaml

@@ -0,0 +1,28 @@
+name: Run unit tests
+
+on:
+  workflow_call:
+    inputs:
+      branch:
+        description: 'The branch to test'
+        required: true
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container: docker:latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: Run unit tests
+        # Note: We can add a test step after the wait-for-postgres.sh script
+        run: |
+          docker-compose -f docker-compose.test.yml up --build -d --force-recreate
+          docker exec -w /code poi-service-backend ls -la
+          docker exec -w /code poi-service-backend ./wait-for-postgres.sh
+          docker-compose -f docker-compose.test.yml down -v -t 0

Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.11
+ARG ENVIRONMENT=release
 
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
+FROM postgis/postgis:14-3.3 AS builder
+# This docker image is based on the bullseye operating system
+# See: https://github.com/postgis/docker-postgis/blob/master/14-3.3/Dockerfile
 
 # Install libraries needed for GeoDjango and PostGIS
 # See https://docs.djangoproject.com/en/3.2/ref/contrib/gis/install/geolibs/
@@ -11,7 +12,13 @@ RUN apt-get update && apt-get install -y \
   gdal-bin
 
 # Install Postgres client to check liveness of the database
-RUN apt-get update && apt-get install -y postgresql-client
+RUN apt-get install -y postgresql-client
+
+# Install osm2pgsql to load the osm-data into the database
+RUN apt-get install -y osm2pgsql
+
+# Install Python and a dependency for psycopg2
+RUN apt-get install -y python3 python3-pip libpq-dev
 
 # Install Poetry as the package manager for this application
 RUN pip install poetry
@@ -21,6 +28,52 @@ WORKDIR /code
 # Install Python dependencies separated from the
 # run script to enable Docker caching
 ADD pyproject.toml /code
-RUN poetry install --no-interaction --no-ansi --no-dev
+# Install all dependencies
+RUN poetry install
+
+# Install CURL for healthcheck
+RUN apt-get update && apt-get install -y curl
+
+# Expose Django port, DO NOT EXPOSE THE DATABASE PORT!
+EXPOSE 8000
 
 COPY . /code
+
+# Used for preheating the correct POIs from overpass turbo.
+ARG LOCATION=Hamburg
+ENV LOCATION=${LOCATION}
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+ENV POSTGRES_NAME=poiservice-db
+ENV POSTGRES_USER=poiservice-user
+ENV POSTGRES_PASSWORD=poiservice-password
+ENV POSTGRES_DB=poiservice-db
+ENV POSTGRES_HOST=localhost
+ENV POSTGRES_PORT=5432
+
+ENV HEALTHCHECK_TOKEN=healthcheck-token
+
+# Use this argument to invalidate the cache of subsequent steps.
+ARG CACHE_DATE=1970-01-01
+
+FROM builder AS test
+# Only run postgres to execute tests
+CMD "./run-test.sh"
+
+FROM builder AS dev
+ENV DJANGO_DEBUG_MODE=True
+# Run the development server
+RUN ./run-preheating.sh
+HEALTHCHECK --interval=60s --timeout=10s --retries=5 --start-period=10s \
+  CMD curl --fail http://localhost:8000/healthcheck?token=healthcheck-token || exit 1
+CMD "./run-dev.sh"
+
+FROM builder AS production
+ENV DJANGO_DEBUG_MODE=False
+# Preheat our database, by running migrations and pre-loading data
+RUN ./run-preheating.sh
+HEALTHCHECK --interval=60s --timeout=10s --retries=5 --start-period=10s \
+  CMD curl --fail http://localhost:8000/healthcheck?token=healthcheck-token || exit 1
+CMD "./run-prod.sh"