Replace safety with pip audit (#645)

* chore: 🤖 ensure poetry uses the local python version

* feat: 🎸 upgrade the dependencies of the libs

also: use poetry 1.2.2, and replace safety with pip-audit

* fix: 🐛 remove dependency to old pymongo[srv] version

it's now included in mongoengine. we had to use this dependency to use
mongo URL with "+srv"

* feat: 🎸 upgrade all the other projects

replacing safety with pip-audit, upgrading the dependencies, after
rewriting poetry.lock with poetry 1.2

* feat: 🎸 upgrade docker images

* chore: 🤖 upgrade poetry in dockerfiles

* chore: 🤖 fix dependencies issues in workers

* ci: 🎡 try to fix pip-audit

see pypa/pip-audit#84 (comment)
in particular

* feat: 🎸 update docker images

Author: severo

.github/workflows/_e2e_tests.yml

@@ -10,7 +10,7 @@ on:
         type: string
 env:
   python-version: 3.9.6
-  poetry-version: 1.1.13
+  poetry-version: 1.2.2
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
   working-directory: e2e

.github/workflows/_quality-python.yml

@@ -8,17 +8,14 @@ on:
       working-directory:
         required: true
         type: string
-      safety-exceptions:
-        required: false
-        type: string
       is-datasets-worker:
         required: false
         type: boolean
 env:
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
   python-version: "3.9.6"
-  poetry-version: "1.1.13"
+  poetry-version: "1.2.2"
 jobs:
   code-quality:
     defaults:
@@ -55,5 +52,9 @@ jobs:
         run: poetry run mypy tests src
       - name: Run bandit
         run: poetry run bandit -r src
-      - name: Run safety
-        run: poetry run safety check ${{ inputs.safety-exceptions }}
+      - name: Run pip-audit (datasets worker)
+        if: ${{ inputs.is-datasets-worker == true }}
+        run: bash -c "poetry run pip-audit -r <(poetry export -f requirements.txt --with dev | sed '/^requests==2.28.1 ;/,+2 d' | sed '/^kenlm @/d' | sed '/^trec-car-tools @/d')"
+      - name: Run pip-audit
+        if: ${{ inputs.is-datasets-worker == false }}
+        run: bash -c 'poetry run pip-audit -r <(poetry export -f requirements.txt --with dev)'

.github/workflows/_unit-tests-python.yml

@@ -16,7 +16,7 @@ env:
   POETRY_VIRTUALENVS_IN_PROJECT: false
   mongo-port: "27017"
   python-version: "3.9.6"
-  poetry-version: "1.1.13"
+  poetry-version: "1.2.2"
 jobs:
   unit-tests:
     defaults:

.github/workflows/w-first_rows.yml

@@ -17,7 +17,6 @@ jobs:
     uses: ./.github/workflows/_quality-python.yml
     with:
       working-directory: workers/first_rows
-      safety-exceptions: ""
       is-datasets-worker: true
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml

.github/workflows/w-splits.yml

@@ -17,7 +17,6 @@ jobs:
     uses: ./.github/workflows/_quality-python.yml
     with:
       working-directory: workers/splits
-      safety-exceptions: ""
       is-datasets-worker: true
   unit-tests:
     uses: ./.github/workflows/_unit-tests-python.yml

chart/docker-images.yaml

@@ -2,15 +2,15 @@
   "dockerImage": {
     "reverseProxy": "docker.io/nginx:1.20",
     "jobs": {
-      "mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-dfa89b1"
+      "mongodbMigration": "huggingface/datasets-server-jobs-mongodb_migration:sha-b6d4c8a"
     },
     "services": {
-      "admin": "huggingface/datasets-server-services-admin:sha-dfa89b1",
-      "api": "huggingface/datasets-server-services-api:sha-dfa89b1"
+      "admin": "huggingface/datasets-server-services-admin:sha-b6d4c8a",
+      "api": "huggingface/datasets-server-services-api:sha-b6d4c8a"
     },
     "workers": {
-      "splits": "huggingface/datasets-server-workers-splits:sha-a0d80a3",
-      "firstRows": "huggingface/datasets-server-workers-first_rows:sha-319bbb8"
+      "splits": "huggingface/datasets-server-workers-splits:sha-4a48536",
+      "firstRows": "huggingface/datasets-server-workers-first_rows:sha-4a48536"
     }
   }
 }

e2e/Makefile

@@ -17,6 +17,7 @@ DOCKER_COMPOSE := ../tools/docker-compose-datasets-server-from-remote-images.yml
 DOCKER_IMAGES := ../chart/docker-images.yaml
 
 include ../tools/Python.mk
+include ../tools/PythonAudit.mk
 include ../tools/PythonTest.mk
 include ../tools/DockerRemoteImages.mk
 include ../tools/Docker.mk