Support python 3.10 and update Docker image (#4324)

* test with python 3.10 * fix dependencies * Docker file to python3.10 * fix tests * fix ci * fix docker image
GeotrekCE · Oct 8, 2024 · 34bee06 · 34bee06
1 parent f678a11
commit 34bee06
Show file tree

Hide file tree

Showing 14 changed files with 114 additions and 108 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -45,8 +45,7 @@ jobs:
 
     strategy:
       matrix:
-        os: ['bionic-3.8', 'focal-3.8']
-        python-version: ['3.8']
+        os: ['bionic-3.8', 'focal-3.8', 'jammy-3.10']
         tests-env: ['tests', 'tests_nds']
 
     env:
@@ -79,10 +78,10 @@ jobs:
         path: |
           ./venv
 
-        key: pip-${{ matrix.python-version }}-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }}
+        key: pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }}
         restore-keys: |
-          pip-${{ matrix.python-version }}-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }}
-          pip-${{ matrix.python-version }}-${{ matrix.os }}
+          pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt', './dev-requirements.txt', './docs/requirements.txt') }}
+          pip-${{ matrix.os }}
 
     - name: Prepare test env
       run: |
@@ -113,12 +112,20 @@ jobs:
            libvips
         apt-get install -y --no-install-recommends postgis
 
+    - name: Install python venv for bionic
+      if: ${{ matrix.os == 'bionic-3.8'}}
+      run: |
+          python3.8 -m venv ./venv
+
+    - name: Install python venv
+      if: ${{ matrix.os != 'bionic-3.8'}}
+      run: |
+          python3 -m venv ./venv
+
     - name: Install python dependencies
       run: |
-        python3.8 -m venv ./venv
-        ./venv/bin/pip3 install --upgrade pip wheel setuptools -c requirements.txt -c dev-requirements.txt
-        ./venv/bin/pip3 install -r requirements.txt -U
-        ./venv/bin/pip3 install -r dev-requirements.txt -U
+        ./venv/bin/pip3 install --upgrade pip wheel setuptools pip-tools -c requirements.txt -c dev-requirements.txt
+        ./venv/bin/pip-sync requirements.txt dev-requirements.txt
 
     - name: Create test required directories
       run: |
@@ -149,8 +156,8 @@ jobs:
         ./venv/bin/coverage xml -o coverage.xml
 
     - name: Save coverage
+      if: ${{ matrix.os != 'bionic-3.8'}}
       uses: actions/upload-artifact@v4
-      if: ${{ matrix.os != 'bionic-3.8' }}
       with:
         name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
         path: coverage.xml
@@ -161,16 +168,15 @@ jobs:
     needs: [ test ]
     strategy:
       matrix:
-        os: ['focal-3.8']
-        python-version: ['3.8']
+        os: ['focal-3.8', 'jammy-3.10']
         tests-env: ['tests', 'tests_nds']
 
     env:
       ENV: ${{ matrix.tests-env }}
       OS: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
             name: coverage_${{ matrix.os }}_${{ matrix.tests-env }}
@@ -199,49 +205,43 @@ jobs:
         with:
           image: "geotrek:latest"
 
-  build_deb_18_04:
-      name: Build 18.04 package
+  build_deb:
+      name: Build debian package
       runs-on: ubuntu-latest
-      steps:
-        - uses: actions/checkout@v3
-        - name: Prepare debian 18.04 versioning
-          run: |
-            grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu18.04~dev'$GITHUB_RUN_ID'/' debian/changelog
-            sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu18.04\2) bionic;/' debian/changelog
-
-        - name: Building package debian 18.04
-          run: |
-            DISTRO=ubuntu:bionic make build_deb
-            cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/
-
-        - name: Archive package artifact
-          uses: actions/upload-artifact@v4
-          with:
-            name: debian-18-04
-            path: |
-              /home/runner/work/Geotrek-admin/*.deb
-
-  build_deb_20_04:
-    name: Build 20.04 package
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Prepare debian 20.04 versioning
-        run: |
-          sed -i 's/+dev/.ubuntu20.04~dev'$GITHUB_RUN_ID'/' debian/changelog
-          sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu20.04\2) focal;/' debian/changelog
+      strategy:
+          matrix:
+              os: ['bionic', 'focal', 'jammy']
+              include:
+                  - os: 'bionic'
+                    code: '18.04'
+                  - os: 'focal'
+                    code: '20.04'
+                  - os: 'jammy'
+                    code: '22.04'
+
+      env:
+          OS: ${{ matrix.os }}
+          CODE: ${{ matrix.code }}
+          DISTRO: ubuntu:${{ matrix.os }}
 
-      - name: Building package debian 18.04
-        run: |
-          DISTRO=ubuntu:focal make build_deb
-          cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/
-
-      - name: Archive package artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: debian-20-04
-          path: |
-            /home/runner/work/Geotrek-admin/*.deb
+      steps:
+          - uses: actions/checkout@v4
+          - name: Prepare versioning
+            run: |
+                grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu'$CODE'~dev'$GITHUB_RUN_ID'/' debian/changelog
+                sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu'$CODE'\2) $OS;/' debian/changelog
+
+          - name: Building package
+            run: |
+              make build_deb
+              cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/
+
+          - name: Archive package artifact
+            uses: actions/upload-artifact@v4
+            with:
+              name: ${{ matrix.os }}
+              path: |
+                /home/runner/work/Geotrek-admin/*.deb
 
   e2e_docker_image:
     name: Tests E2E docker
@@ -329,7 +329,7 @@ jobs:
   e2e_deb_20_04:
     name: Tests E2E 20.04
     runs-on: ubuntu-20.04
-    needs: [ build_deb_20_04 ]
+    needs: [ build_deb ]
 
     env:
       CYPRESS_BASE_URL: http://geotrek.local
@@ -352,7 +352,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v4
         with:
-          name: debian-20-04
+          name: 'focal'
 
       - name: Simulate install.sh procedure
         run: |
@@ -407,25 +407,26 @@ jobs:
           path: |
             /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4
             /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png
+
   deploy:
     name: Publish (on release only)
     runs-on: ubuntu-latest
-    needs: [ test, e2e_docker_image, build_deb_18_04, e2e_deb_20_04 ]
+    needs: [ test, e2e_docker_image, build_deb, e2e_deb_20_04 ]
     if: ${{ github.event_name == 'release' && github.event.action == 'created' }}
     steps:
       - uses: actions/checkout@v3
       - name: Download 18.04 debian artifact
         uses: actions/download-artifact@v4
         with:
-          name: debian-18-04
+          name: bionic
       - name: Download 20.04 debian artifact
         uses: actions/download-artifact@v4
         with:
           name: debian-20-04
       - name: Download docker image
         uses: ishworkh/docker-image-artifact-download@v1
         with:
-          image: "geotrek:latest"
+          image: focal
       - name: Attach debian packages as release binaries
         uses: skx/github-action-publish-binaries@master
         env:

diff --git a/debian/conffiles b/debian/conffiles
@@ -1,4 +1,4 @@
-opt/geotrek-admin/var/conf/env.in
-opt/geotrek-admin/var/conf/nginx.conf.in
-opt/geotrek-admin/var/conf/gunicorn-geotrek.conf.py.in
-opt/geotrek-admin/var/conf/gunicorn-geotrek_api.conf.py.in
+/opt/geotrek-admin/var/conf/env.in
+/opt/geotrek-admin/var/conf/nginx.conf.in
+/opt/geotrek-admin/var/conf/gunicorn-geotrek.conf.py.in
+/opt/geotrek-admin/var/conf/gunicorn-geotrek_api.conf.py.in
diff --git a/debian/control b/debian/control
@@ -10,9 +10,9 @@ Build-Depends:
  debhelper (>= 9),
  dh-virtualenv (>= 1.0),
  git,
- python3.8 (>= 3.8),
- python3.8-venv (>= 3.8),
- python3.8-dev (>= 3.8),
+ python3.8 | python3.10,
+ python3.8-venv | python3.10-venv,
+ python3.8-dev | python3.10-dev,
  libgdal-dev (>= 2.2),
  libffi-dev (>= 3),
  libxml2-dev (>= 2.9.2),
@@ -37,6 +37,6 @@ Depends:
  screamshotter (>= 2.0.9),
  screamshotter (<< 3.0.0),
  convertit,
- python3.8 (>= 3.8),
+ python3.8 | python3.10,
  libvips
 Description: Manage and promote your trails and tourist content and activities.
diff --git a/debian/geotrek-admin.triggers b/debian/geotrek-admin.triggers
@@ -1,9 +1,8 @@
 # Register interest in Python interpreter changes; and
 # don't make the Python package dependent on the virtualenv package
 # processing (noawait)
-interest-noawait /usr/bin/python3.5
-interest-noawait /usr/bin/python3.6
 interest-noawait /usr/bin/python3.8
+interest-noawait /usr/bin/python3.10
 
 # Also provide a symbolic trigger for all dh-virtualenv packages
 interest dh-virtualenv-interpreter-update
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -8,7 +8,7 @@ asgiref==3.8.1
     # via
     #   -c requirements.txt
     #   django
-backports-zoneinfo==0.2.1
+backports-zoneinfo==0.2.1 ; python_version < "3.10"
     # via
     #   -c requirements.txt
     #   django

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,6 +5,8 @@ x-images:
       context: .
       dockerfile: ./docker/Dockerfile
       target: dev
+      args:
+          BASE_IMAGE: focal
     user: ${UID:-1000}:${GID:-1000}
     env_file:
       - .env

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,33 +1,32 @@
-FROM ubuntu:focal as base
-
-ENV PYTHONBUFFERED 1
-ENV DEBIAN_FRONTEND noninteractive
-ENV ENV prod
-ENV SERVER_NAME "localhost"
-# If POSTGRES_HOST is empty, entrypoint will set it to the IP of the docker host in the container
-ENV POSTGRES_HOST ""
-ENV POSTGRES_PORT "5432"
-ENV POSTGRES_USER "geotrek"
-ENV POSTGRES_PASSWORD "geotrek"
-ENV POSTGRES_DB "geotrekdb"
-ENV REDIS_HOST "redis"
-ENV CONVERSION_HOST "convertit"
-ENV CAPTURE_HOST "screamshotter"
-ENV CUSTOM_SETTINGS_FILE "/opt/geotrek-admin/var/conf/custom.py"
-ENV TZ UTC
+ARG BASE_IMAGE=jammy
+
+FROM ubuntu:${BASE_IMAGE} AS base
+
+ENV PYTHONBUFFERED=1
+ENV DEBIAN_FRONTEND=noninteractive
+ENV ENV=prod
+ENV SERVER_NAME="localhost"
+ENV POSTGRES_HOST="host.docker.internal"
+ENV POSTGRES_PORT="5432"
+ENV POSTGRES_USER="geotrek"
+ENV POSTGRES_DB="geotrekdb"
+ENV REDIS_HOST="redis"
+ENV CONVERSION_HOST="convertit"
+ENV CAPTURE_HOST="screamshotter"
+ENV CUSTOM_SETTINGS_FILE="/opt/geotrek-admin/var/conf/custom.py"
+ENV TZ=UTC
 
 WORKDIR /opt/geotrek-admin
 RUN mkdir -p /opt/geotrek-admin/var/log /opt/geotrek-admin/var/cache
-RUN adduser geotrek --disabled-password && chown geotrek:geotrek -R /opt
+RUN useradd -M -s /bin/false geotrek && chown geotrek:geotrek -R /opt
 
 # Install postgis because raster2pgsl is required by manage.py loaddem
 RUN apt-get update -qq && apt-get install -y -qq  \
-    python3.8 \
+    python3 \
     ca-certificates \
     gettext \
     postgresql-client \
     tzdata \
-    netcat \
     gdal-bin \
     binutils \
     libproj-dev \
@@ -38,13 +37,12 @@ RUN apt-get update -qq && apt-get install -y -qq  \
     curl \
     software-properties-common \
     shared-mime-info \
+    libmagic1 \
     fonts-liberation \
     libssl-dev \
     libfreetype6-dev \
     libxml2-dev \
     libxslt-dev \
-    libcairo2 \
-    libpango1.0-0 \
     libpangocairo-1.0-0 \
     libgdk-pixbuf2.0-dev \
     libffi-dev \
@@ -58,36 +56,35 @@ COPY --chown=geotrek:geotrek docker/scripts/* /usr/local/bin/
 ENTRYPOINT ["/bin/sh", "-e", "/usr/local/bin/entrypoint.sh"]
 EXPOSE 8000
 
-FROM base as build
+FROM base AS build
 
 USER root
 
 RUN apt-get update -qq && apt-get install -y -qq  \
     git \
-    python3.8-dev \
-    python3.8-venv \
+    python3-dev \
+    python3-venv \
     build-essential \
     libpq-dev &&\
     apt-get clean all && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/*
 
 USER geotrek
-RUN python3.8 -m venv /opt/venv
+RUN python3 -m venv /opt/venv
 RUN /opt/venv/bin/pip install --no-cache-dir -U pip setuptools wheel
 COPY requirements.txt requirements.txt
 RUN /opt/venv/bin/pip install --no-cache-dir -r requirements.txt -U
 
-FROM build as dev
+FROM build AS dev
 
+ENV ENV=dev
 COPY dev-requirements.txt dev-requirements.txt
 RUN /opt/venv/bin/pip install --no-cache-dir -r dev-requirements.txt
 COPY ./docs/requirements.txt doc-requirements.txt
 RUN /opt/venv/bin/pip install --no-cache-dir -r doc-requirements.txt
 
 CMD ["./manage.py", "runserver", "0.0.0.0:8000"]
 
-FROM base as prod
-
-ENV ENV prod
+FROM base AS prod
 
 COPY --chown=geotrek:geotrek --from=build /opt/venv /opt/venv
 COPY --chown=geotrek:geotrek geotrek/ geotrek/

diff --git a/docker/Dockerfile.debian.builder b/docker/Dockerfile.debian.builder
@@ -29,6 +29,12 @@ RUN env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --tool='apt-get -
 COPY . ./
 WORKDIR /dpkg-build
 
+RUN if test "$(lsb_release -cs)" = 'jammy' ; then \
+      sed -i 's/python3.8/python3.10/g' debian/rules; \
+    elif test "$(lsb_release -cs)" = 'noble' ; then \
+      sed -i 's/python3.8/python3.12/g' debian/rules; \
+    fi
+
 RUN sed -i -re "1s/..UNRELEASED/.ubuntu$(lsb_release -rs)) $(lsb_release -cs)/" debian/changelog \
     && chmod a-x debian/geotrek.* \
     && dpkg-buildpackage -us -uc -b && mkdir -p /dpkg && cp -pl /geotrek[-_]* /dpkg \