From 66b4a15b3726e6124a7e4f2ee9f7038d85aa7fcb Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Tue, 3 Dec 2024 18:32:08 -0400 Subject: [PATCH 1/5] Update `dev` group of dependencies --- api/pdm.lock | 2 +- api/pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/pdm.lock b/api/pdm.lock index 5b356af53d6..c3c05794b8a 100644 --- a/api/pdm.lock +++ b/api/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "overrides", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:f3221f00e34af7caec4730447e3d2237e735ae558a0ca320f55fadf30a7cc831" +content_hash = "sha256:86e04ec218fb1f4ffe4e553a2ac20a19c41dd8bc8b3f25d3f19e9319cbf3cea5" [[metadata.targets]] requires_python = "==3.12.*" diff --git a/api/pyproject.toml b/api/pyproject.toml index dccfef9fe3d..959658c5213 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -50,9 +50,9 @@ overrides = [ ] dev = [ "debugpy >= 1.8.9, <2", - "ipython >=8.30.0, <9", + "ipython >=8.30, <9", "pgcli >=4.1, <5", - "remote-pdb >=2.1.0, <3", + "remote-pdb >=2.1, <3", "setuptools >=75.6, <76", # FIXME: Should be removed when Python build issues are resolved, # FIXME: since we don't explicitly use this library. From a8f822ae7c70ca5929816910a9b279a57e8b2764 Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Tue, 3 Dec 2024 19:59:12 -0400 Subject: [PATCH 2/5] Bump PDM to 2.21 in the api/ and indexer_worker/ --- api/Dockerfile | 6 +++--- indexer_worker/Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 54be3f5cf33..f8e273119d1 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -19,11 +19,11 @@ ENV PIP_NO_COLOR=1 # - Create a virtualenv inside `/venv` # - Install PDM to install Python dependencies RUN apt-get update \ - && apt-get install -yqq --no-install-recommends \ - python3-dev \ + && apt-get install -yqq --no-install-recommends python3-dev \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* \ - && pip install pdm~=2.19 + && pip install --upgrade pip \ + && pip install pdm~=2.21 # Copy subpackages from additional build-context 'packages' # hadolint ignore=DL3022 diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 95eefba0f1b..4ed6498baba 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -18,11 +18,11 @@ ENV PIP_NO_COLOR=1 # - Install system packages needed for building Python dependencies # - Install PDM to install Python dependencies RUN apt-get update \ - && apt-get install -yqq --no-install-recommends \ - python3-dev \ + && apt-get install -yqq --no-install-recommends python3-dev \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* \ - && pip install pdm~=2.14 + && pip install --upgrade pip \ + && pip install pdm~=2.21 # Copy the Pipenv files into the container COPY pyproject.toml pdm.lock / From 5a3a8b296153b8fb39bc0dccdcaf58936d05f763 Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Thu, 5 Dec 2024 11:14:58 -0400 Subject: [PATCH 3/5] Add and use OV_PDM_VERSION environment variable --- api/Dockerfile | 5 ++++- api/compose.yml | 1 + indexer_worker/Dockerfile | 5 ++++- indexer_worker/compose.yml | 1 + justfile | 1 + 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index f8e273119d1..29338de9046 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -10,6 +10,9 @@ ARG API_PY_VERSION FROM docker.io/python:${API_PY_VERSION} AS builder +# Get the version from the main justfile +ARG OV_PDM_VERSION + # Container optimizations ENV PYTHONUNBUFFERED=1 ENV PIP_NO_CACHE_DIR=1 @@ -23,7 +26,7 @@ RUN apt-get update \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* \ && pip install --upgrade pip \ - && pip install pdm~=2.21 + && pip install pdm~=$OV_PDM_VERSION # Copy subpackages from additional build-context 'packages' # hadolint ignore=DL3022 diff --git a/api/compose.yml b/api/compose.yml index f88eee7295d..9ff821b3a47 100644 --- a/api/compose.yml +++ b/api/compose.yml @@ -16,6 +16,7 @@ services: args: # Automatically inferred from env vars, unless specified - SEMANTIC_VERSION=${SEMANTIC_VERSION:-v1.0.0} - API_PY_VERSION + - OV_PDM_VERSION=${OV_PDM_VERSION} - PDM_INSTALL_ARGS=--dev image: openverse-api:${API_PDM_HASH:-latest} pull_policy: never diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 4ed6498baba..b56885eca89 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -10,6 +10,9 @@ ARG INDEXER_WORKER_PY_VERSION FROM docker.io/python:${INDEXER_WORKER_PY_VERSION} AS builder +# Get the version from the main justfile +ARG OV_PDM_VERSION + # Container optimizations ENV PYTHONUNBUFFERED=1 ENV PIP_NO_CACHE_DIR=1 @@ -22,7 +25,7 @@ RUN apt-get update \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* \ && pip install --upgrade pip \ - && pip install pdm~=2.21 + && pip install pdm~=$OV_PDM_VERSION # Copy the Pipenv files into the container COPY pyproject.toml pdm.lock / diff --git a/indexer_worker/compose.yml b/indexer_worker/compose.yml index 42e94142d89..dffa261ff64 100644 --- a/indexer_worker/compose.yml +++ b/indexer_worker/compose.yml @@ -8,6 +8,7 @@ services: target: indexer_worker args: # Automatically inferred from env vars, unless specified - INDEXER_WORKER_PY_VERSION + - OV_PDM_VERSION=${OV_PDM_VERSION} - PDM_INSTALL_ARGS=--dev ports: - "50282:8003" diff --git a/justfile b/justfile index a1b43276426..a063ae69332 100644 --- a/justfile +++ b/justfile @@ -232,6 +232,7 @@ env: DOCKER_FILE := "-f docker-compose.yml" EXEC_DEFAULTS := if IS_CI == "" { "" } else { "-T" } +export OV_PDM_VERSION := env_var_or_default("OV_PDM_VERSION", "2.21") export CATALOG_PY_VERSION := `just catalog/py-version` export CATALOG_AIRFLOW_VERSION := `just catalog/airflow-version` export INDEXER_WORKER_PY_VERSION := `just indexer_worker/py-version` From 997b3fba46d97d7a79e6a1a322e231c2ae0f6dd2 Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Thu, 5 Dec 2024 11:15:52 -0400 Subject: [PATCH 4/5] Convert single quotes to double quotes --- api/Dockerfile | 2 +- indexer_worker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 29338de9046..2f7123d3eae 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -36,7 +36,7 @@ COPY --from=packages openverse-attribution /packages/python/openverse-attributio COPY pyproject.toml pdm.lock / # Pass additional arguments when installing Python packages with PDM -ARG PDM_INSTALL_ARGS='--no-editable' +ARG PDM_INSTALL_ARGS="--no-editable" # Install Python dependencies into a new virtualenv RUN pdm install --check --frozen-lockfile $PDM_INSTALL_ARGS diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index b56885eca89..9210ac1f036 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -31,7 +31,7 @@ RUN apt-get update \ COPY pyproject.toml pdm.lock / # Pass additional arguments when installing Python packages with PDM -ARG PDM_INSTALL_ARGS='--no-editable' +ARG PDM_INSTALL_ARGS="--no-editable" # Install Python dependencies into a new virtualenv RUN pdm install --check --frozen-lockfile $PDM_INSTALL_ARGS From 7720b729965c24eb096b2df6289f4772e85d414c Mon Sep 17 00:00:00 2001 From: Krystle Salazar Date: Thu, 5 Dec 2024 11:37:26 -0400 Subject: [PATCH 5/5] Add OV_PDM_VERSION to ci_cd.yml workflow --- .github/workflows/ci_cd.yml | 3 +++ justfile | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 000aa6f818a..59212efa2fe 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -192,6 +192,7 @@ jobs: build-contexts: ${{ matrix.build-contexts }} build-args: | SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} + OV_PDM_VERSION=${{ steps.prepare-build-args.outputs.ov_pdm_version }} CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }} CATALOG_AIRFLOW_VERSION=${{ steps.prepare-build-args.outputs.catalog_airflow_version }} INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }} @@ -367,6 +368,7 @@ jobs: cache-to: type=gha,scope=indexer_worker_dev build-args: | INDEXER_WORKER_PY_VERSION=${{ steps.prepare-build-args.outputs.indexer_worker_py_version }} + OV_PDM_VERSION=${{ steps.prepare-build-args.outputs.ov_pdm_version }} PDM_INSTALL_ARGS=--dev - name: Run tests @@ -484,6 +486,7 @@ jobs: build-args: | SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }} + OV_PDM_VERSION=${{ steps.prepare-build-args.outputs.ov_pdm_version }} PDM_INSTALL_ARGS=--dev - name: Start Catalog diff --git a/justfile b/justfile index a063ae69332..4676d7bbe36 100644 --- a/justfile +++ b/justfile @@ -248,6 +248,7 @@ export HOST_NETWORK_ADDRESS := if os() == "macos" { "host.docker.internal" } els versions: #!/usr/bin/env bash cat <