From 34bee0680361012df5be393681d34e5251ce1b5a Mon Sep 17 00:00:00 2001 From: Jean-Etienne Castagnede Date: Tue, 8 Oct 2024 15:58:05 +0200 Subject: [PATCH] Support python 3.10 and update Docker image (#4324) * test with python 3.10 * fix dependencies * Docker file to python3.10 * fix tests * fix ci * fix docker image --- .github/workflows/test.yml | 119 ++++++++++++++++--------------- debian/conffiles | 8 +-- debian/control | 8 +-- debian/geotrek-admin.triggers | 3 +- dev-requirements.txt | 2 +- docker-compose.yml | 2 + docker/Dockerfile | 57 +++++++-------- docker/Dockerfile.debian.builder | 6 ++ docker/scripts/entrypoint.sh | 1 - docs/changelog.rst | 1 + geotrek/common/parsers.py | 2 +- geotrek/settings/env_tests.py | 5 +- requirements.txt | 7 +- setup.py | 1 + 14 files changed, 114 insertions(+), 108 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36dd80c108..ac063ea7d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,8 +45,7 @@ jobs: strategy: matrix: - os: ['bionic-3.8', 'focal-3.8'] - python-version: ['3.8'] + os: ['bionic-3.8', 'focal-3.8', 'jammy-3.10'] tests-env: ['tests', 'tests_nds'] env: @@ -79,10 +78,10 @@ jobs: path: | ./venv - key: pip-${{ matrix.python-version }}-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }} + key: pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }} restore-keys: | - pip-${{ matrix.python-version }}-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }} - pip-${{ matrix.python-version }}-${{ matrix.os }} + pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }} + pip-${{ matrix.os }} - name: Prepare test env run: | @@ -113,12 +112,20 @@ jobs: libvips apt-get install -y --no-install-recommends postgis + - name: Install python venv for bionic + if: ${{ matrix.os == 'bionic-3.8'}} + run: | + python3.8 -m venv ./venv + + - name: Install python venv + if: ${{ matrix.os != 'bionic-3.8'}} + run: | + python3 -m venv ./venv + - name: Install python dependencies run: | - python3.8 -m venv ./venv - ./venv/bin/pip3 install --upgrade pip wheel setuptools -c requirements.txt -c dev-requirements.txt - ./venv/bin/pip3 install -r requirements.txt -U - ./venv/bin/pip3 install -r dev-requirements.txt -U + ./venv/bin/pip3 install --upgrade pip wheel setuptools pip-tools -c requirements.txt -c dev-requirements.txt + ./venv/bin/pip-sync requirements.txt dev-requirements.txt - name: Create test required directories run: | @@ -149,8 +156,8 @@ jobs: ./venv/bin/coverage xml -o coverage.xml - name: Save coverage + if: ${{ matrix.os != 'bionic-3.8'}} uses: actions/upload-artifact@v4 - if: ${{ matrix.os != 'bionic-3.8' }} with: name: coverage_${{ matrix.os }}_${{ matrix.tests-env }} path: coverage.xml @@ -161,8 +168,7 @@ jobs: needs: [ test ] strategy: matrix: - os: ['focal-3.8'] - python-version: ['3.8'] + os: ['focal-3.8', 'jammy-3.10'] tests-env: ['tests', 'tests_nds'] env: @@ -170,7 +176,7 @@ jobs: OS: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: coverage_${{ matrix.os }}_${{ matrix.tests-env }} @@ -199,49 +205,43 @@ jobs: with: image: "geotrek:latest" - build_deb_18_04: - name: Build 18.04 package + build_deb: + name: Build debian package runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Prepare debian 18.04 versioning - run: | - grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu18.04~dev'$GITHUB_RUN_ID'/' debian/changelog - sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu18.04\2) bionic;/' debian/changelog - - - name: Building package debian 18.04 - run: | - DISTRO=ubuntu:bionic make build_deb - cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/ - - - name: Archive package artifact - uses: actions/upload-artifact@v4 - with: - name: debian-18-04 - path: | - /home/runner/work/Geotrek-admin/*.deb - - build_deb_20_04: - name: Build 20.04 package - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Prepare debian 20.04 versioning - run: | - sed -i 's/+dev/.ubuntu20.04~dev'$GITHUB_RUN_ID'/' debian/changelog - sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu20.04\2) focal;/' debian/changelog + strategy: + matrix: + os: ['bionic', 'focal', 'jammy'] + include: + - os: 'bionic' + code: '18.04' + - os: 'focal' + code: '20.04' + - os: 'jammy' + code: '22.04' + + env: + OS: ${{ matrix.os }} + CODE: ${{ matrix.code }} + DISTRO: ubuntu:${{ matrix.os }} - - name: Building package debian 18.04 - run: | - DISTRO=ubuntu:focal make build_deb - cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/ - - - name: Archive package artifact - uses: actions/upload-artifact@v4 - with: - name: debian-20-04 - path: | - /home/runner/work/Geotrek-admin/*.deb + steps: + - uses: actions/checkout@v4 + - name: Prepare versioning + run: | + grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu'$CODE'~dev'$GITHUB_RUN_ID'/' debian/changelog + sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu'$CODE'\2) $OS;/' debian/changelog + + - name: Building package + run: | + make build_deb + cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/ + + - name: Archive package artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }} + path: | + /home/runner/work/Geotrek-admin/*.deb e2e_docker_image: name: Tests E2E docker @@ -329,7 +329,7 @@ jobs: e2e_deb_20_04: name: Tests E2E 20.04 runs-on: ubuntu-20.04 - needs: [ build_deb_20_04 ] + needs: [ build_deb ] env: CYPRESS_BASE_URL: http://geotrek.local @@ -352,7 +352,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/download-artifact@v4 with: - name: debian-20-04 + name: 'focal' - name: Simulate install.sh procedure run: | @@ -407,17 +407,18 @@ jobs: path: | /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4 /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png + deploy: name: Publish (on release only) runs-on: ubuntu-latest - needs: [ test, e2e_docker_image, build_deb_18_04, e2e_deb_20_04 ] + needs: [ test, e2e_docker_image, build_deb, e2e_deb_20_04 ] if: ${{ github.event_name == 'release' && github.event.action == 'created' }} steps: - uses: actions/checkout@v3 - name: Download 18.04 debian artifact uses: actions/download-artifact@v4 with: - name: debian-18-04 + name: bionic - name: Download 20.04 debian artifact uses: actions/download-artifact@v4 with: @@ -425,7 +426,7 @@ jobs: - name: Download docker image uses: ishworkh/docker-image-artifact-download@v1 with: - image: "geotrek:latest" + image: focal - name: Attach debian packages as release binaries uses: skx/github-action-publish-binaries@master env: diff --git a/debian/conffiles b/debian/conffiles index c8520f7bd7..14fdfa65c6 100644 --- a/debian/conffiles +++ b/debian/conffiles @@ -1,4 +1,4 @@ -opt/geotrek-admin/var/conf/env.in -opt/geotrek-admin/var/conf/nginx.conf.in -opt/geotrek-admin/var/conf/gunicorn-geotrek.conf.py.in -opt/geotrek-admin/var/conf/gunicorn-geotrek_api.conf.py.in +/opt/geotrek-admin/var/conf/env.in +/opt/geotrek-admin/var/conf/nginx.conf.in +/opt/geotrek-admin/var/conf/gunicorn-geotrek.conf.py.in +/opt/geotrek-admin/var/conf/gunicorn-geotrek_api.conf.py.in diff --git a/debian/control b/debian/control index e204839507..6e0772b938 100644 --- a/debian/control +++ b/debian/control @@ -10,9 +10,9 @@ Build-Depends: debhelper (>= 9), dh-virtualenv (>= 1.0), git, - python3.8 (>= 3.8), - python3.8-venv (>= 3.8), - python3.8-dev (>= 3.8), + python3.8 | python3.10, + python3.8-venv | python3.10-venv, + python3.8-dev | python3.10-dev, libgdal-dev (>= 2.2), libffi-dev (>= 3), libxml2-dev (>= 2.9.2), @@ -37,6 +37,6 @@ Depends: screamshotter (>= 2.0.9), screamshotter (<< 3.0.0), convertit, - python3.8 (>= 3.8), + python3.8 | python3.10, libvips Description: Manage and promote your trails and tourist content and activities. diff --git a/debian/geotrek-admin.triggers b/debian/geotrek-admin.triggers index 9410395a07..20ffdad4b5 100644 --- a/debian/geotrek-admin.triggers +++ b/debian/geotrek-admin.triggers @@ -1,9 +1,8 @@ # Register interest in Python interpreter changes; and # don't make the Python package dependent on the virtualenv package # processing (noawait) -interest-noawait /usr/bin/python3.5 -interest-noawait /usr/bin/python3.6 interest-noawait /usr/bin/python3.8 +interest-noawait /usr/bin/python3.10 # Also provide a symbolic trigger for all dh-virtualenv packages interest dh-virtualenv-interpreter-update diff --git a/dev-requirements.txt b/dev-requirements.txt index de7b257152..d75269bdc2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,7 +8,7 @@ asgiref==3.8.1 # via # -c requirements.txt # django -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.10" # via # -c requirements.txt # django diff --git a/docker-compose.yml b/docker-compose.yml index 92fa06dbb9..1f1eea41e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ x-images: context: . dockerfile: ./docker/Dockerfile target: dev + args: + BASE_IMAGE: focal user: ${UID:-1000}:${GID:-1000} env_file: - .env diff --git a/docker/Dockerfile b/docker/Dockerfile index a4238717e4..0bf0139a0f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,33 +1,32 @@ -FROM ubuntu:focal as base - -ENV PYTHONBUFFERED 1 -ENV DEBIAN_FRONTEND noninteractive -ENV ENV prod -ENV SERVER_NAME "localhost" -# If POSTGRES_HOST is empty, entrypoint will set it to the IP of the docker host in the container -ENV POSTGRES_HOST "" -ENV POSTGRES_PORT "5432" -ENV POSTGRES_USER "geotrek" -ENV POSTGRES_PASSWORD "geotrek" -ENV POSTGRES_DB "geotrekdb" -ENV REDIS_HOST "redis" -ENV CONVERSION_HOST "convertit" -ENV CAPTURE_HOST "screamshotter" -ENV CUSTOM_SETTINGS_FILE "/opt/geotrek-admin/var/conf/custom.py" -ENV TZ UTC +ARG BASE_IMAGE=jammy + +FROM ubuntu:${BASE_IMAGE} AS base + +ENV PYTHONBUFFERED=1 +ENV DEBIAN_FRONTEND=noninteractive +ENV ENV=prod +ENV SERVER_NAME="localhost" +ENV POSTGRES_HOST="host.docker.internal" +ENV POSTGRES_PORT="5432" +ENV POSTGRES_USER="geotrek" +ENV POSTGRES_DB="geotrekdb" +ENV REDIS_HOST="redis" +ENV CONVERSION_HOST="convertit" +ENV CAPTURE_HOST="screamshotter" +ENV CUSTOM_SETTINGS_FILE="/opt/geotrek-admin/var/conf/custom.py" +ENV TZ=UTC WORKDIR /opt/geotrek-admin RUN mkdir -p /opt/geotrek-admin/var/log /opt/geotrek-admin/var/cache -RUN adduser geotrek --disabled-password && chown geotrek:geotrek -R /opt +RUN useradd -M -s /bin/false geotrek && chown geotrek:geotrek -R /opt # Install postgis because raster2pgsl is required by manage.py loaddem RUN apt-get update -qq && apt-get install -y -qq \ - python3.8 \ + python3 \ ca-certificates \ gettext \ postgresql-client \ tzdata \ - netcat \ gdal-bin \ binutils \ libproj-dev \ @@ -38,13 +37,12 @@ RUN apt-get update -qq && apt-get install -y -qq \ curl \ software-properties-common \ shared-mime-info \ + libmagic1 \ fonts-liberation \ libssl-dev \ libfreetype6-dev \ libxml2-dev \ libxslt-dev \ - libcairo2 \ - libpango1.0-0 \ libpangocairo-1.0-0 \ libgdk-pixbuf2.0-dev \ libffi-dev \ @@ -58,26 +56,27 @@ COPY --chown=geotrek:geotrek docker/scripts/* /usr/local/bin/ ENTRYPOINT ["/bin/sh", "-e", "/usr/local/bin/entrypoint.sh"] EXPOSE 8000 -FROM base as build +FROM base AS build USER root RUN apt-get update -qq && apt-get install -y -qq \ git \ - python3.8-dev \ - python3.8-venv \ + python3-dev \ + python3-venv \ build-essential \ libpq-dev &&\ apt-get clean all && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/* USER geotrek -RUN python3.8 -m venv /opt/venv +RUN python3 -m venv /opt/venv RUN /opt/venv/bin/pip install --no-cache-dir -U pip setuptools wheel COPY requirements.txt requirements.txt RUN /opt/venv/bin/pip install --no-cache-dir -r requirements.txt -U -FROM build as dev +FROM build AS dev +ENV ENV=dev COPY dev-requirements.txt dev-requirements.txt RUN /opt/venv/bin/pip install --no-cache-dir -r dev-requirements.txt COPY ./docs/requirements.txt doc-requirements.txt @@ -85,9 +84,7 @@ RUN /opt/venv/bin/pip install --no-cache-dir -r doc-requirements.txt CMD ["./manage.py", "runserver", "0.0.0.0:8000"] -FROM base as prod - -ENV ENV prod +FROM base AS prod COPY --chown=geotrek:geotrek --from=build /opt/venv /opt/venv COPY --chown=geotrek:geotrek geotrek/ geotrek/ diff --git a/docker/Dockerfile.debian.builder b/docker/Dockerfile.debian.builder index e33e743792..69dd19c7c7 100644 --- a/docker/Dockerfile.debian.builder +++ b/docker/Dockerfile.debian.builder @@ -29,6 +29,12 @@ RUN env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --tool='apt-get - COPY . ./ WORKDIR /dpkg-build +RUN if test "$(lsb_release -cs)" = 'jammy' ; then \ + sed -i 's/python3.8/python3.10/g' debian/rules; \ + elif test "$(lsb_release -cs)" = 'noble' ; then \ + sed -i 's/python3.8/python3.12/g' debian/rules; \ + fi + RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \ && chmod a-x debian/geotrek.* \ && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /geotrek[-_]* /dpkg \ diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index 964f343b61..1293c3dabe 100755 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -26,7 +26,6 @@ fi # Activate venv . /opt/venv/bin/activate -# Defaults POSTGRES_HOST to Docker host IP export POSTGRES_HOST=${POSTGRES_HOST:-`ip route | grep default | sed 's/.* \([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\) .*/\1/'`} # Defaults SECRET_KEY to a random value diff --git a/docs/changelog.rst b/docs/changelog.rst index cd77951d72..996b1c67e2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,7 @@ CHANGELOG **Maintenance** +- Docker image use now fully tested Ubuntu Jammy and python3.10 - Use new crispy form - Bump mapentity to 8.10.0. Mapentity login page has been improved, take care about your logo-login.png size. diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 45c03a6ec7..e5157f5f2a 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -13,7 +13,7 @@ import xlrd import xml.etree.ElementTree as ET from functools import reduce -from collections import Iterable +from collections.abc import Iterable from time import sleep from PIL import Image, UnidentifiedImageError diff --git a/geotrek/settings/env_tests.py b/geotrek/settings/env_tests.py index 0c3514bbe1..936f27117c 100644 --- a/geotrek/settings/env_tests.py +++ b/geotrek/settings/env_tests.py @@ -64,8 +64,8 @@ def __getitem__(self, item): if os.path.exists(TMP_DIR): shutil.rmtree(TMP_DIR) os.makedirs(TMP_DIR) -SESSIONS_DIR = os.path.join(TMP_DIR, 'sessions') -os.makedirs(SESSIONS_DIR) +SESSION_FILE_PATH = TemporaryDirectory(dir=TMP_DIR).name +os.makedirs(SESSION_FILE_PATH) LOGGING['loggers']['']['handlers'] = ('log_file', ) LOGGING['handlers']['log_file']['level'] = 'INFO' @@ -75,7 +75,6 @@ def __getitem__(self, item): MOBILE_TILES_PATH = TemporaryDirectory(dir=TMP_DIR).name # sync mobile tile path DATA_TEMP_DIR = TemporaryDirectory(dir=TMP_DIR).name # data temp dir use by django-large-image REDIS_URL = f"redis://{os.getenv('REDIS_HOST', 'localhost')}:{os.getenv('REDIS_PORT', '6379')}/1" # celery broker url -SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" PASSWORD_HASHERS = [ "django.contrib.auth.hashers.MD5PasswordHasher", ] diff --git a/requirements.txt b/requirements.txt index a8723c9e54..534af31c44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,11 +14,12 @@ async-timeout==4.0.3 # via redis attrs==24.2.0 # via fiona -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.10" # via # celery # django # djangorestframework + # geotrek (setup.py) # kombu beautifulsoup4==4.12.3 # via mapentity @@ -242,7 +243,7 @@ markupsafe==2.1.5 # via jinja2 mbutil==0.3.0 # via landez -numpy==1.23.4 +numpy==1.24.4 # via # large-image # large-image-source-vips @@ -326,7 +327,7 @@ rjsmin==1.2.1 # via django-compressor ruamel-yaml==0.18.6 # via drf-yasg -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via ruamel-yaml sentry-sdk==2.14.0 # via geotrek (setup.py) diff --git a/setup.py b/setup.py index 34fca1b696..b34032010c 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ def run(self): scripts=['manage.py'], install_requires=[ 'Django==4.2.*', + 'backports-zoneinfo;python_version<"3.10"', # not directly needed but required to make it worked with 3.10 'mapentity', 'chardet', 'cairosvg',