fix: github actions

Author: joaquimds

.github/workflows/lint.yml

@@ -4,20 +4,19 @@ name: Lint
 
 on:
   pull_request:
-  workflow_call:
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: black Lint
+      - name: Run black
         uses: psf/black@stable
         with:
           options: "--check --diff"
           src: "."
           version: "22.8.0"
-      - name: isort Lint
+      - name: Run isort
         uses: isort/isort-action@master
-      - name: flake8 Lint
+      - name: Run flake8
         uses: py-actions/flake8@v2

.github/workflows/tests.yml

@@ -5,12 +5,12 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
   pull_request:
 
 jobs:
   tests:
     runs-on: ubuntu-latest
+    environment: testing
     services:
       postgres:
         image: postgres
@@ -23,30 +23,45 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
+    container:
+      image: python:3.12
+      env:
+        DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
+        CACHE_FILE: /tmp/meep
+        POETRY_VIRTUALENVS_CREATE: "false"
+        MAPIT_URL: https://mapit.mysociety.org/
+        MAPIT_API_KEY: "not_a_key"
+        TEST_AIRTABLE_BASE_ID: ${{ vars.TEST_AIRTABLE_BASE_ID }}
+        TEST_AIRTABLE_TABLE_NAME: ${{ vars.TEST_AIRTABLE_TABLE_NAME }}
+        TEST_AIRTABLE_API_KEY: ${{ secrets.TEST_AIRTABLE_API_KEY }}
+        SECRET_KEY: keyboardcat
     steps:
-      - name: checkout repo content
+      - name: Checkout repo content
         uses: actions/checkout@v2
-
+      - name: Install linux dependencies
+        run: |
+          curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null
+          echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list
+          apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok less
+      - name: Install poetry
+        run: curl -sSL https://install.python-poetry.org | python3 -
+      - name: Install python dependencies
+        run: ~/.local/bin/poetry export --with dev -f requirements.txt --output requirements.txt && pip install -r requirements.txt
+      - name: Start server
+        run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 &
+      - name: Start ngrok tunnelling
+        run: |
+          ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }}
+          ngrok http 8000 --log=stdout > ngrok.log &
+      - name: Extract ngrok URL
+        run: BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}') && echo "BASE_URL=$BASE_URL" > .env
       - name: Run django tests
-        uses: ./.github/actions/in-container
-        env:
-          DATABASE_URL: postgres://postgres:postgres@postgres/postgres
-          CACHE_FILE: /tmp/meep
-          SECRET_KEY: keyboardcat
-        with:
-          run: |
-            script/bootstrap
-            script/test --coverage
-
-      # do this inside the docker container otherwise the paths don't work
-      - name: generate coverage xml
-        uses: ./.github/actions/in-container
-        with:
-          run: coverage xml
-
-      # do this outside the docker container otherwise can't get repo details
-      - name: upload code coverage
+        run: cat .env && coverage run --source=. --branch manage.py test
+      - name: Generate coverage xml
+        run: coverage xml
+      - name: Upload code coverage
         run: |
           less coverage.xml
           pip install codecov
           codecov
+

hub/middleware.py

@@ -24,7 +24,7 @@ def record_last_seen_middleware(get_response):
     def process_request(request):
         if request.user.is_authenticated:
             user = request.user
-            props = UserProperties.objects.get_or_create(user=user)
+            props, _ = UserProperties.objects.get_or_create(user=user)
             last_seen = request.session.get("last_seen", None)
             yesterday = now().replace(hour=0, minute=0) - one_day
             if last_seen is None or last_seen < yesterday.timestamp():
@@ -33,11 +33,13 @@ def process_request(request):
                 props.save()
 
     if iscoroutinefunction(get_response):
+
         async def middleware(request: HttpRequest):
             await sync_to_async(process_request)(request)
             return await get_response(request)
 
     else:
+
         def middleware(request: HttpRequest):
             process_request(request)
             return get_response(request)
@@ -51,13 +53,15 @@ def logic(request):
         return WhiteNoiseMiddleware(get_response)(request)
 
     if iscoroutinefunction(get_response):
+
         async def middleware(request: HttpRequest):
             response = logic(request)
             if isawaitable(response):
                 response = await response
             return response
 
     else:
+
         def middleware(request: HttpRequest):
             return logic(request)
 
@@ -81,6 +85,7 @@ def exception_handler(error: Exception, request: HttpRequest):
         setattr(request, USER_OR_ERROR_KEY, user_or_error)
 
     if iscoroutinefunction(get_response):
+
         async def middleware(request: HttpRequest):
             try:
                 return await gqlauth_middleware(request)
@@ -89,10 +94,11 @@ async def middleware(request: HttpRequest):
             return await get_response(request)
 
     else:
+
         def middleware(request: HttpRequest):
             try:
                 return gqlauth_middleware(request)
-            except Exception as e:
+            except Exception:
                 exception_handler(request)
             return get_response(request)
 

hub/views/core.py

@@ -1,11 +1,10 @@
 from django.db import connection
 from django.db.utils import OperationalError
-from django.http import JsonResponse
+from django.http import HttpResponse, JsonResponse
 from django.views.generic import TemplateView
 
 from hub.mixins import TitleMixin
 from hub.models import Area, DataSet
-from django.http import HttpResponse
 
 
 async def async_healthcheck(request):

local_intelligence_hub/settings.py

@@ -10,7 +10,6 @@
 https://docs.djangoproject.com/en/4.1/ref/settings/
 """
 
-import socket
 from datetime import timedelta
 from pathlib import Path
 
@@ -28,7 +27,7 @@
     SCHEDULED_UPDATE_SECONDS_DELAY=(int, 3),
     DEBUG=(bool, False),
     ALLOWED_HOSTS=(list, []),
-    CORS_ALLOWED_ORIGINS=(list, ['http://localhost:3000']),
+    CORS_ALLOWED_ORIGINS=(list, ["http://localhost:3000"]),
     GOOGLE_ANALYTICS=(str, ""),
     GOOGLE_SITE_VERIFICATION=(str, ""),
     TEST_AIRTABLE_BASE_ID=(str, ""),

pyproject.toml

@@ -15,6 +15,7 @@ requests-cache = "^0.9.6"
 Pillow = "^10.2.0"
 python-magic = "^0.4.27"
 tqdm = "^4.64.1"
+pandas = "^2.2.1"
 openpyxl = "^3.0.10"
 mysoc-dataset = "^0.3.0"
 django-jsonform = "^2.15.0"
@@ -34,7 +35,6 @@ whitenoise = "^6.6.0"
 setuptools = "^69.1.1"
 uvicorn = "^0.27.1"
 strawberry-graphql = {extras = ["asgi"], version = "^0.220.0"}
-pandas = "^2.2.1"
 
 [tool.poetry.dev-dependencies]
 black = "^22.8.0"