Back to development #4411
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Test CI | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - 'docs/**' | |
| release: | |
| types: | |
| - created | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| POSTGRES_DB: ci_test | |
| POSTGRES_PASSWORD: ci_test | |
| POSTGRES_USER: ci_test | |
| POSTGRES_HOST: 127.0.0.1 | |
| SERVER_NAME: geotrek.local | |
| SECRET_KEY: test-test-test | |
| CONVERSION_HOST: localhost | |
| CAPTURE_HOST: localhost | |
| REDIS_HOST: 127.0.0.1 | |
| VAR_DIR: /home/runner/work/Geotrek-admin/Geotrek-admin/var | |
| LANG: C.UTF-8 | |
| LANGUAGES: en fr | |
| LANGUAGE_CODE: en | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| jobs: | |
| test: | |
| name: Python Unittests | |
| runs-on: ubuntu-latest | |
| container: makinacorpus/geodjango:${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: ['bionic-3.8', 'focal-3.8', 'jammy-3.10'] | |
| tests-env: ['tests', 'tests_nds'] | |
| env: | |
| ENV: ${{ matrix.tests-env }} | |
| OS: ${{ matrix.os }} | |
| POSTGRES_HOST: postgres | |
| REDIS_HOST: redis | |
| services: | |
| postgres: | |
| image: postgis/postgis:12-2.5 | |
| env: | |
| POSTGRES_DB: ci_test | |
| POSTGRES_PASSWORD: ci_test | |
| POSTGRES_USER: ci_test | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| redis: | |
| image: redis:5.0-alpine | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./venv | |
| key: pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }} | |
| restore-keys: | | |
| pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }}-${{ hashFiles('./docs/requirements.txt') }} | |
| pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }}-${{ hashFiles('./dev-requirements.txt') }} | |
| pip-${{ matrix.os }}-${{ hashFiles('./requirements.txt') }} | |
| pip-${{ matrix.os }} | |
| - name: Prepare test env | |
| run: | | |
| apt-get update -q | |
| apt-get -qq -y install \ | |
| unzip \ | |
| gettext \ | |
| binutils \ | |
| libproj-dev \ | |
| gdal-bin \ | |
| sudo \ | |
| less \ | |
| curl \ | |
| git \ | |
| iproute2 \ | |
| software-properties-common \ | |
| shared-mime-info \ | |
| fonts-liberation \ | |
| libssl-dev \ | |
| libfreetype6-dev \ | |
| libxml2-dev \ | |
| libxslt-dev \ | |
| libcairo2 \ | |
| libpango1.0-0 \ | |
| libpangocairo-1.0-0 \ | |
| libgdk-pixbuf2.0-dev \ | |
| libffi-dev \ | |
| libvips | |
| apt-get install -y --no-install-recommends postgis | |
| - name: Install python venv for bionic | |
| if: ${{ matrix.os == 'bionic-3.8'}} | |
| run: | | |
| python3.8 -m venv ./venv | |
| - name: Install python venv | |
| if: ${{ matrix.os != 'bionic-3.8'}} | |
| run: | | |
| python3 -m venv ./venv | |
| - name: Install python dependencies | |
| run: | | |
| ./venv/bin/pip3 install --upgrade pip wheel setuptools pip-tools -c requirements.txt -c dev-requirements.txt | |
| ./venv/bin/pip-sync requirements.txt dev-requirements.txt | |
| - name: Create test required directories | |
| run: | | |
| mkdir -p ./var/log ./var/cache/sessions | |
| - name: Prepare environment | |
| run: | | |
| mkdir -p $VAR_DIR/conf/extra_static $VAR_DIR/tmp | |
| ./venv/bin/python3 ./manage.py collectstatic --clear --noinput --verbosity=0 | |
| ./venv/bin/python3 ./manage.py compilemessages | |
| - name: Check missing migrations | |
| run: | | |
| ENV=dev ./venv/bin/python3 ./manage.py makemigrations --check | |
| - name: Check running migrations | |
| run: | | |
| ENV=dev ./venv/bin/python3 ./manage.py migrate | |
| - name: Launch Celery | |
| run: | | |
| ./venv/bin/celery -A geotrek worker -c 1 & | |
| - name: Unit test | |
| run: | | |
| ./venv/bin/coverage run --parallel-mode --concurrency=multiprocessing ./manage.py test --parallel -v 3 | |
| ./venv/bin/coverage combine | |
| ./venv/bin/coverage xml -o coverage.xml | |
| - name: Save coverage | |
| if: ${{ matrix.os != 'bionic-3.8'}} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage_${{ matrix.os }}_${{ matrix.tests-env }} | |
| path: coverage.xml | |
| upload_coverage: | |
| name: Upload coverage to codecov | |
| runs-on: ubuntu-latest | |
| needs: [ test ] | |
| strategy: | |
| matrix: | |
| os: ['focal-3.8', 'jammy-3.10'] | |
| tests-env: ['tests', 'tests_nds'] | |
| env: | |
| ENV: ${{ matrix.tests-env }} | |
| OS: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage_${{ matrix.os }}_${{ matrix.tests-env }} | |
| path: . | |
| - uses: codecov/codecov-action@v4 | |
| with: | |
| files: ./coverage.xml | |
| env_vars: OS,ENV | |
| token: ${{ secrets.CODECOV_TOKEN }} # not usually required for public repos | |
| fail_ci_if_error: true # optional (default = false) | |
| build_docker_image: | |
| name: Build docker image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: false | |
| builder: ${{ steps.buildx.outputs.name }} | |
| tags: "geotrek:latest" | |
| load: true | |
| file: ./docker/Dockerfile | |
| - name: Upload image | |
| uses: ishworkh/container-image-artifact-upload@v2.0.0 | |
| with: | |
| image: "geotrek:latest" | |
| build_deb: | |
| name: Build debian package | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| os: ['bionic', 'focal', 'jammy'] | |
| include: | |
| - os: 'bionic' | |
| code: '18.04' | |
| - os: 'focal' | |
| code: '20.04' | |
| - os: 'jammy' | |
| code: '22.04' | |
| env: | |
| OS: ${{ matrix.os }} | |
| CODE: ${{ matrix.code }} | |
| DISTRO: ubuntu:${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare versioning | |
| run: | | |
| grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' VERSION || sed -i 's/+dev/.ubuntu'$CODE'~dev'$GITHUB_RUN_ID'/' debian/changelog | |
| sed -i 's/geotrek-admin (\([0-9]\+\.[0-9]\+\.[0-9]\+\)\(.*\)) RELEASED;/geotrek-admin (\1.ubuntu'$CODE'\2) '$OS';/' debian/changelog | |
| - name: Building package | |
| run: | | |
| make build_deb | |
| cp ./dpkg/*.deb /home/runner/work/Geotrek-admin/ | |
| - name: Archive package artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.os }} | |
| path: | | |
| /home/runner/work/Geotrek-admin/*.deb | |
| e2e_docker_image: | |
| name: Tests E2E docker | |
| runs-on: ubuntu-20.04 | |
| needs: [build_docker_image] | |
| env: | |
| CYPRESS_BASE_URL: http://geotrek.local | |
| CYPRESS_CACHE_FOLDER: ~/cypress/cache | |
| services: | |
| postgres: | |
| image: postgis/postgis:12-2.5 | |
| env: | |
| POSTGRES_DB: ci_test | |
| POSTGRES_PASSWORD: ci_test | |
| POSTGRES_USER: ci_test | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download image | |
| uses: ishworkh/container-image-artifact-download@v2.0.0 | |
| with: | |
| image: "geotrek:latest" | |
| - name: Launch service | |
| run: | | |
| docker tag geotrek:latest geotrekce/admin:latest | |
| cp docker/install/.env.dist .env | |
| cp docker/install/docker-compose.yml docker-compose.yml | |
| echo 'LANGUAGES=en fr' >> .env | |
| echo 'LANGUAGE_CODE=en' >> .env | |
| sudo sed -i 's/localhost/geotrek.local/' .env | |
| sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts | |
| mkdir -p var/conf/extra_static | |
| cp cypress/custom.py var/conf/custom.py | |
| echo 'POSTGRES_USER=ci_test' >> .env | |
| echo 'POSTGRES_PASSWORD=ci_test' >> .env | |
| echo 'POSTGRES_DB=ci_test' >> .env | |
| docker compose run web update.sh | |
| make load_data | |
| make load_test_integration | |
| docker compose up -d | |
| - name: Prepare nginx | |
| run: | | |
| sudo service apache2 stop | |
| sudo apt-get purge apache2 apache2-utils apache2.2-bin | |
| sudo apt-get autoremove | |
| sudo apt-get install -q -y nginx | |
| sudo unlink /etc/nginx/sites-enabled/default | |
| sudo mkdir /var/www/geotrek -p | |
| ls -als | |
| pwd | |
| sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/media /var/www/geotrek/media | |
| sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/var/static /var/www/geotrek/static | |
| sudo ln -s /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/nginx.conf /etc/nginx/sites-enabled/geotrek.conf | |
| sudo systemctl restart nginx | |
| - name: install Cypress | |
| run: | | |
| cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress | |
| npm ci | |
| npm list | |
| echo "Waiting for container..." | |
| while ! nc -z "127.0.0.1" "8001"; do | |
| sleep 0.1 | |
| done | |
| - name: E2E test | |
| run: | | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564 | |
| continue-on-error: false | |
| - name: Archive results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cypress-docker | |
| path: | | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4 | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png | |
| e2e_deb_20_04: | |
| name: Tests E2E 20.04 | |
| runs-on: ubuntu-20.04 | |
| needs: [ build_deb ] | |
| env: | |
| CYPRESS_BASE_URL: http://geotrek.local | |
| CYPRESS_CACHE_FOLDER: ~/cypress/cache | |
| services: | |
| postgres: | |
| image: postgis/postgis:12-2.5 | |
| env: | |
| POSTGRES_DB: ci_test | |
| POSTGRES_PASSWORD: ci_test | |
| POSTGRES_USER: ci_test | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: 'focal' | |
| - name: Simulate install.sh procedure | |
| run: | | |
| sudo service apache2 stop | |
| sudo apt-get purge apache2 apache2-utils apache2.2-bin | |
| sudo apt-get autoremove | |
| echo "deb [arch=amd64] https://packages.geotrek.fr/ubuntu $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/geotrek.list | |
| wget -O- "https://packages.geotrek.fr/geotrek.gpg.key" | sudo apt-key add - | |
| sudo apt-get -q update | |
| echo "geotrek-admin geotrek-admin/LANGUAGES string en fr" >> .debian_settings | |
| echo "geotrek-admin geotrek-admin/POSTGRES_USER string $POSTGRES_USER" >> .debian_settings | |
| echo "geotrek-admin geotrek-admin/POSTGRES_DB string $POSTGRES_DB" >> .debian_settings | |
| echo "geotrek-admin geotrek-admin/POSTGRES_PASSWORD string $POSTGRES_PASSWORD" >> .debian_settings | |
| echo "geotrek-admin geotrek-admin/SERVER_NAME string geotrek.local" >> .debian_settings | |
| mkdir -p /opt/geotrek-admin/var/conf/ | |
| cp /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/custom.py /opt/geotrek-admin/var/conf/custom.py | |
| sudo sed -i 's/127.0.0.1 localhost/127.0.0.1 localhost geotrek.local/' /etc/hosts | |
| sudo debconf-set-selections .debian_settings | |
| - name: Install package | |
| run: | | |
| sudo unlink /etc/nginx/sites-enabled/default | |
| sudo apt-get install /home/runner/work/Geotrek-admin/Geotrek-admin/*.deb || exit 0; | |
| sudo systemctl restart nginx | |
| - name: Load Data | |
| run: | | |
| sudo geotrek loaddata minimal | |
| sudo geotrek loaddata cirkwi | |
| sudo geotrek loaddata basic | |
| sudo geotrek loaddata test-integration | |
| for dir in `ls -d /opt/geotrek-admin/lib/python3*/site-packages/geotrek/*/fixtures/upload`; do | |
| cd $dir > /dev/null | |
| sudo cp -r * /opt/geotrek-admin/var/media/upload/ | |
| done | |
| - name: Install cypress | |
| run: | | |
| cd /home/runner/work/Geotrek-admin/Geotrek-admin/cypress | |
| npm ci | |
| npm list | |
| - name: E2E test | |
| run: | | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/node_modules/.bin/cypress run -P /home/runner/work/Geotrek-admin/Geotrek-admin/cypress --record --key 64a5a9b3-9869-4a2f-91e4-e3cd27c2f564 | |
| continue-on-error: false | |
| - name: Archive results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cypress-20.04 | |
| path: | | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/videos/*.mp4 | |
| /home/runner/work/Geotrek-admin/Geotrek-admin/cypress/screenshots/*.png | |
| deploy: | |
| name: Publish (on release only) | |
| runs-on: ubuntu-latest | |
| needs: [ test, e2e_docker_image, build_deb, e2e_deb_20_04 ] | |
| if: ${{ github.event_name == 'release' && github.event.action == 'created' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download 18.04 debian artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: bionic | |
| - name: Download 20.04 debian artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: focal | |
| - name: Download docker image | |
| uses: ishworkh/container-image-artifact-download@v2.0.0 | |
| with: | |
| image: 'geotrek:latest' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_LOGIN }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Publish Docker image (v2 and latest) | |
| run: | | |
| if [[ "${{ github.ref }}" == *"dev"* ]]; then | |
| export DOCKER_TAG=dev | |
| else | |
| docker tag geotrek:latest geotrekce/admin:$(cat VERSION) | |
| docker push geotrekce/admin:$(cat VERSION) | |
| export DOCKER_TAG=latest | |
| fi | |
| docker tag geotrek:latest geotrekce/admin:$DOCKER_TAG | |
| docker push geotrekce/admin:$DOCKER_TAG | |
| - name: Install SSH key | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| known_hosts: ${{ secrets.SSH_KNOWN_HOSTS }} | |
| - name: Publish debian packages | |
| run: | | |
| if [[ "${{ github.ref }}" == *"dev"* ]]; then | |
| export DEB_COMPONENT=dev | |
| else | |
| export DEB_COMPONENT=main | |
| fi | |
| echo "${{ github.ref }} : Publishing as $DEB_COMPONENT package" | |
| scp -P ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no /home/runner/work/Geotrek-admin/Geotrek-admin/geotrek-admin_*_amd64.deb ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }}:/srv/packages/incoming/$DEB_COMPONENT/ | |
| if [[ "${{ github.ref }}" == *"dev"* ]]; then | |
| ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_dev -C /srv/packages | |
| ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_dev -C /srv/packages | |
| else | |
| ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make bionic_main -C /srv/packages | |
| ssh -p ${{ secrets.SSH_PORT }} -o StrictHostKeyChecking=no ${{ secrets.SSH_USER }}@${{ secrets.SSH_HOST }} make focal_main -C /srv/packages | |
| fi | |
| - name: Download 22.04 debian artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: jammy | |
| - name: Attach debian packages as release binaries | |
| uses: skx/github-action-publish-binaries@master | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| args: '*.deb' |