diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 70d2d13c8..2cfada1fb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,21 +3,21 @@ name: Lint on: + push: pull_request: - workflow_call: jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: black Lint + - name: Run black uses: psf/black@stable with: options: "--check --diff" src: "." version: "22.8.0" - - name: isort Lint + - name: Run isort uses: isort/isort-action@master - - name: flake8 Lint + - name: Run flake8 uses: py-actions/flake8@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aad36f63e..bbba34ae4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,7 @@ on: jobs: tests: runs-on: ubuntu-latest + environment: testing services: postgres: image: postgres @@ -23,30 +24,45 @@ jobs: --health-retries 5 ports: - 5432:5432 + container: + image: python:3.12 + env: + DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres + CACHE_FILE: /tmp/meep + POETRY_VIRTUALENVS_CREATE: "false" + MAPIT_URL: https://mapit.mysociety.org/ + MAPIT_API_KEY: "not_a_key" + TEST_AIRTABLE_BASE_ID: ${{ vars.TEST_AIRTABLE_BASE_ID }} + TEST_AIRTABLE_TABLE_NAME: ${{ vars.TEST_AIRTABLE_TABLE_NAME }} + TEST_AIRTABLE_API_KEY: ${{ secrets.TEST_AIRTABLE_API_KEY }} + SECRET_KEY: keyboardcat steps: - - name: checkout repo content + - name: Checkout repo content uses: actions/checkout@v2 - + - name: Install linux dependencies + run: | + curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null + echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | tee /etc/apt/sources.list.d/ngrok.list + apt-get update && apt-get install -y binutils gdal-bin libproj-dev ngrok + - name: Install poetry + run: curl -sSL https://install.python-poetry.org | python3 - + - name: Install python dependencies + run: ~/.local/bin/poetry export --with dev -f requirements.txt --output requirements.txt && pip install -r requirements.txt + - name: Start server + run: gunicorn local_intelligence_hub.asgi:application -k uvicorn.workers.UvicornWorker --bind 0.0.0.0:8000 & + - name: Start ngrok tunnelling + run: | + ngrok authtoken ${{ secrets.NGROK_AUTHTOKEN }} + ngrok http 8000 --log=stdout > ngrok.log & + - name: Extract ngrok URL + run: BASE_URL=$(cat ngrok.log | grep 'url=' | awk -F= '{print $NF}') && echo "BASE_URL=$BASE_URL" > .env - name: Run django tests - uses: ./.github/actions/in-container - env: - DATABASE_URL: postgres://postgres:postgres@postgres/postgres - CACHE_FILE: /tmp/meep - SECRET_KEY: keyboardcat - with: - run: | - script/bootstrap - script/test --coverage - - # do this inside the docker container otherwise the paths don't work - - name: generate coverage xml - uses: ./.github/actions/in-container - with: - run: coverage xml - - # do this outside the docker container otherwise can't get repo details - - name: upload code coverage + run: cat .env && coverage run --source=. --branch manage.py test + - name: Generate coverage xml + run: coverage xml + - name: Upload code coverage run: | less coverage.xml pip install codecov codecov + diff --git a/hub/middleware.py b/hub/middleware.py index 854e90496..bb445d501 100644 --- a/hub/middleware.py +++ b/hub/middleware.py @@ -24,7 +24,7 @@ def record_last_seen_middleware(get_response): def process_request(request): if request.user.is_authenticated: user = request.user - props = UserProperties.objects.get_or_create(user=user) + props, _ = UserProperties.objects.get_or_create(user=user) last_seen = request.session.get("last_seen", None) yesterday = now().replace(hour=0, minute=0) - one_day if last_seen is None or last_seen < yesterday.timestamp(): @@ -33,11 +33,13 @@ def process_request(request): props.save() if iscoroutinefunction(get_response): + async def middleware(request: HttpRequest): await sync_to_async(process_request)(request) return await get_response(request) else: + def middleware(request: HttpRequest): process_request(request) return get_response(request) @@ -51,6 +53,7 @@ def logic(request): return WhiteNoiseMiddleware(get_response)(request) if iscoroutinefunction(get_response): + async def middleware(request: HttpRequest): response = logic(request) if isawaitable(response): @@ -58,6 +61,7 @@ async def middleware(request: HttpRequest): return response else: + def middleware(request: HttpRequest): return logic(request) @@ -81,6 +85,7 @@ def exception_handler(error: Exception, request: HttpRequest): setattr(request, USER_OR_ERROR_KEY, user_or_error) if iscoroutinefunction(get_response): + async def middleware(request: HttpRequest): try: return await gqlauth_middleware(request) @@ -89,10 +94,11 @@ async def middleware(request: HttpRequest): return await get_response(request) else: + def middleware(request: HttpRequest): try: return gqlauth_middleware(request) - except Exception as e: + except Exception: exception_handler(request) return get_response(request) diff --git a/hub/views/core.py b/hub/views/core.py index 2cd5c23a7..ece9f88ad 100644 --- a/hub/views/core.py +++ b/hub/views/core.py @@ -1,11 +1,10 @@ from django.db import connection from django.db.utils import OperationalError -from django.http import JsonResponse +from django.http import HttpResponse, JsonResponse from django.views.generic import TemplateView from hub.mixins import TitleMixin from hub.models import Area, DataSet -from django.http import HttpResponse async def async_healthcheck(request): diff --git a/local_intelligence_hub/settings.py b/local_intelligence_hub/settings.py index 7f5efbd6a..9b7ec2b14 100644 --- a/local_intelligence_hub/settings.py +++ b/local_intelligence_hub/settings.py @@ -10,7 +10,6 @@ https://docs.djangoproject.com/en/4.1/ref/settings/ """ -import socket from datetime import timedelta from pathlib import Path @@ -28,7 +27,7 @@ SCHEDULED_UPDATE_SECONDS_DELAY=(int, 3), DEBUG=(bool, False), ALLOWED_HOSTS=(list, []), - CORS_ALLOWED_ORIGINS=(list, ['http://localhost:3000']), + CORS_ALLOWED_ORIGINS=(list, ["http://localhost:3000"]), GOOGLE_ANALYTICS=(str, ""), GOOGLE_SITE_VERIFICATION=(str, ""), TEST_AIRTABLE_BASE_ID=(str, ""), diff --git a/pyproject.toml b/pyproject.toml index 438870289..ab4571a22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ requests-cache = "^0.9.6" Pillow = "^10.2.0" python-magic = "^0.4.27" tqdm = "^4.64.1" +pandas = "^2.2.1" openpyxl = "^3.0.10" mysoc-dataset = "^0.3.0" django-jsonform = "^2.15.0" @@ -34,7 +35,6 @@ whitenoise = "^6.6.0" setuptools = "^69.1.1" uvicorn = "^0.27.1" strawberry-graphql = {extras = ["asgi"], version = "^0.220.0"} -pandas = "^2.2.1" [tool.poetry.dev-dependencies] black = "^22.8.0"