chore: multiple python version support with latest pyspark and hail

.github/labeler.yml

@@ -2,26 +2,26 @@ version: 1
 labels:
   - label: "size-XS"
     size:
-      exclude-files: ["poetry.lock"]
+      exclude-files: ["uv.lock"]
       below: 10
   - label: "size-S"
     size:
-      exclude-files: ["poetry.lock"]
+      exclude-files: ["uv.lock"]
       above: 9
       below: 100
   - label: "size-M"
     size:
-      exclude-files: ["poetry.lock"]
+      exclude-files: ["uv.lock"]
       above: 100
       below: 500
   - label: "size-L"
     size:
-      exclude-files: ["poetry.lock"]
+      exclude-files: ["uv.lock"]
       above: 499
       below: 1000
   - label: "size-XL"
     size:
-      exclude-files: ["poetry.lock"]
+      exclude-files: ["uv.lock"]
       above: 999
   - label: "airflow"
     files:

.github/pull_request_template.md

@@ -25,4 +25,4 @@ add diagrams or images if necessary. It'll help the reviewer_ -->
 - [ ] Did you make sure the branch is up-to-date with the `dev` branch?
 - [ ] Did you write any new necessary tests?
 - [ ] Did you make sure the changes pass local tests (`make test`)?
-- [ ] Did you make sure the changes pass pre-commit rules (e.g `poetry run pre-commit run --all-files`)?
+- [ ] Did you make sure the changes pass pre-commit rules (e.g `uv run pre-commit run --all-files`)?

.github/workflows/artifact.yml

@@ -12,7 +12,7 @@ env:
   REGION: europe-west1
   GAR_LOCATION: europe-west1-docker.pkg.dev/open-targets-genetics-dev
   REPOSITORY: gentropy-app
-  PYTHON_VERSION_DEFAULT: "3.10.8"
+  PYTHON_VERSION_DEFAULT: "3.11.11"
 
 jobs:
   build-push-artifact:
@@ -54,6 +54,7 @@ jobs:
 
       # skip the `v` at the beginning of the tag for docker image tags
       - name: Create a docker tag
+        if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
         id: docker-tag
         shell: bash
         env:
@@ -86,12 +87,8 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION_DEFAULT }}
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
 
       - name: Build and push spark cluster dependencies
         run: |

.github/workflows/pr.yaml

@@ -4,47 +4,43 @@ name: Checks
   pull_request:
 
 env:
-  PYTHON_VERSION_DEFAULT: "3.10.8"
+  PYTHON_VERSION_DEFAULT: "3.11.11"
 
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+      fail-fast: false
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.10.8
+          python-version: ${{ matrix.python-version }}
       - name: Set up Java
         uses: actions/setup-java@v4
         with:
-          java-version: "8"
+          java-version: "11"
           distribution: "temurin"
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
       - name: Load cached venv
-        id: cached-poetry-dependencies
+        id: cached-uv-dependencies
         uses: actions/cache@v4
         with:
           path: .venv
-          key: venv-${{ runner.os }}-${{ env.PYTHON_VERSION_DEFAULT }}-${{ hashFiles('**/poetry.lock') }}
-      - name: Validate project dependencies
-        run: poetry check
+          key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/uv.lock') }}
       - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      - name: Install library
-        run: poetry install --no-interaction
+        if: steps.cached-uv-dependencies.outputs.cache-hit != 'true'
+        run: uv sync --all-groups
       - name: Check dependencies
-        run: poetry run deptry .
+        run: uv run deptry .
       - name: Run tests
-        run: poetry run pytest
+        run: uv run pytest
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

.github/workflows/release.yaml

@@ -2,14 +2,14 @@ name: Release
 
 "on":
   push:
-    branches: ["main", "release/**"]
+    branches: ["main", "release/**", "dev"]
 
 concurrency:
   group: deploy
   cancel-in-progress: false # prevent hickups with semantic-release
 
 env:
-  PYTHON_VERSION_DEFAULT: "3.10.8"
+  PYTHON_VERSION_DEFAULT: "3.11.11"
 
 jobs:
   release:
@@ -40,9 +40,7 @@ jobs:
 
       - name: Python Semantic Release
         id: semrelease
-        # v9.6.0 is required due to the python v3.12 in the newer version of semantic release action which
-        # breaks the poetry build command.
-        uses: python-semantic-release/python-semantic-release@v9.6.0
+        uses: python-semantic-release/python-semantic-release@v9.16.1
         with:
           github_token: ${{ steps.trigger-token.outputs.token }}
 
@@ -121,25 +119,19 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_VERSION_DEFAULT }}
-      - name: Install and configure Poetry
-        uses: snok/install-poetry@v1
-        with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
       - name: Load cached venv
-        id: cached-poetry-dependencies
+        id: cached-dependencies
         uses: actions/cache@v4
         with:
           path: .venv
           key: |
             venv-${{ runner.os }}-\
             ${{ env.PYTHON_VERSION_DEFAULT }}-\
-            ${{ hashFiles('**/poetry.lock') }}
+            ${{ hashFiles('**/uv.lock') }}
       - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      - name: Install library
-        run: poetry install --without tests --no-interaction
+        if: steps.cached-dependencies.outputs.cache-hit != 'true'
+        run: uv sync --group docs
       - name: Publish docs
-        run: poetry run mkdocs gh-deploy --force
+        run: uv run mkdocs gh-deploy --force

.gitignore

@@ -14,3 +14,4 @@ site/
 .coverage*
 wandb/
 hail*.log
+.python-version

.pre-commit-config.yaml

@@ -1,9 +1,8 @@
 default_language_version:
-  python: python3.10
+  python: python3.11
 ci:
   autoupdate_commit_msg: "chore: pre-commit autoupdate"
   autofix_commit_msg: "chore: pre-commit auto fixes [...]"
-  skip: [poetry-lock]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.7.4
@@ -101,9 +100,7 @@ repos:
     rev: 0.5.9
     hooks:
       - id: pydoclint
-
-  - repo: https://github.com/python-poetry/poetry
-    rev: "2.0.0"
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.5.22
     hooks:
-      - id: poetry-check
-      - id: poetry-lock
+      - id: uv-lock

Dockerfile

@@ -1,27 +1,20 @@
 FROM python:3.10-bullseye
 
-RUN apt-get update \
-  && apt-get clean \
-  && apt-get install -y openjdk-11-jdk \
-  && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get clean && \
+    apt-get install -y openjdk-11-jdk && \
+    rm -rf /var/lib/apt/lists/*
 
-ENV POETRY_NO_INTERACTION=1 \
-    POETRY_VIRTUALENVS_IN_PROJECT=1 \
-    POETRY_VIRTUALENVS_CREATE=1 \
-    POETRY_CACHE_DIR=/tmp/poetry_cache \
-    JAVA_HOME=/usr
+ENV JAVA_HOME=/usr
 
-RUN pip install poetry>=2.0.0
+RUN pip install uv
 WORKDIR /app
 
-COPY pyproject.toml poetry.lock ./
+COPY pyproject.toml uv.lock ./
 RUN touch README.md
 
-RUN poetry config installer.max-workers 10
-RUN poetry install --without dev,docs,tests --no-root --no-interaction --no-ansi -vvv && rm -rf $POETRY_CACHE_DIR
+RUN uv sync
 
 COPY src ./src
 
-RUN poetry install --without dev,docs,tests
-
-ENTRYPOINT ["poetry", "run", "gentropy"]
+ENTRYPOINT ["uv", "run", "gentropy"]

Makefile

@@ -1,7 +1,8 @@
+SHELL := /bin/bash
 PROJECT_ID ?= open-targets-genetics-dev
 REGION ?= europe-west1
 APP_NAME ?= $$(cat pyproject.toml | grep -m 1 "name" | cut -d" " -f3 | sed  's/"//g')
-PACKAGE_VERSION ?= $$(poetry version --short)
+PACKAGE_VERSION ?= $(shell grep -m 1 'version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
 # NOTE: git rev-parse will always return the HEAD if it sits in the tag,
 # this way we can distinguish the tag vs branch name
 ifeq ($(shell git rev-parse --abbrev-ref HEAD),HEAD)
@@ -11,7 +12,7 @@ else
 endif
 
 CLEAN_PACKAGE_VERSION := $(shell echo "$(PACKAGE_VERSION)" | tr -cd '[:alnum:]')
-BUCKET_NAME=gs://genetics_etl_python_playground/initialisation/${APP_NAME}/${REF}
+BUCKET_NAME=gs://genetics_etl_python_playground/initialisation
 
 .PHONY: $(shell sed -n -e '/^$$/ { n ; /^[^ .\#][^ ]*:/ { s/:.*$$// ; p ; } ; }' $(MAKEFILE_LIST))
 
@@ -29,37 +30,48 @@ setup-dev: ## Setup development environment
 
 check: ## Lint and format code
 	@echo "Linting API..."
-	@poetry run ruff check src/gentropy .
+	@uv run ruff check src/gentropy .
 	@echo "Linting docstrings..."
-	@poetry run pydoclint --config=pyproject.toml src
-	@poetry run pydoclint --config=pyproject.toml --skip-checking-short-docstrings=true tests
+	@uv run pydoclint --config=pyproject.toml src
+	@uv run pydoclint --config=pyproject.toml --skip-checking-short-docstrings=true tests
 
 test: ## Run tests
 	@echo "Running Tests..."
-	@poetry run pytest
+	@uv run pytest
 
 build-documentation: ## Create local server with documentation
 	@echo "Building Documentation..."
-	@poetry run mkdocs serve
+	@uv run mkdocs serve
 
-create-dev-cluster: build ## Spin up a simple dataproc cluster with all dependencies for development purposes
+sync-cluster-init-script: ## Synchronize the cluster inicialisation actions script to google cloud
+	@echo "Syncing install_dependencies_on_cluster.sh to ${BUCKET_NAME}"
+	@gcloud storage cp utils/install_dependencies_on_cluster.sh ${BUCKET_NAME}/install_dependencies_on_cluster.sh
+
+sync-gentropy-cli-script: ## Synchronize the gentropy cli script
+	@echo "Syncing gentropy cli script to ${BUCKET_NAME}"
+	@gcloud storage cp src/gentropy/cli.py ${BUCKET_NAME}/cli.py
+
+create-dev-cluster: sync-cluster-init-script sync-gentropy-cli-script ## Spin up a simple dataproc cluster with all dependencies for development purposes
+	@echo "Making sure the branch is in sync with remote, so cluster can install gentropy dev version..."
+	@./utils/clean_status.sh || (echo "ERROR: Commit and push or stash local changes, to have up to date cluster"; exit 1)
 	@echo "Creating Dataproc Dev Cluster"
-	@gcloud config set project ${PROJECT_ID}
-	@gcloud dataproc clusters create "ot-genetics-dev-${CLEAN_PACKAGE_VERSION}-$(USER)" \
-		--image-version 2.1 \
+	gcloud config set project ${PROJECT_ID}
+	gcloud dataproc clusters create "ot-genetics-dev-${CLEAN_PACKAGE_VERSION}-$(USER)" \
+		--image-version 2.2 \
 		--region ${REGION} \
-		--master-machine-type n1-standard-16 \
-		--initialization-actions=$(BUCKET_NAME)/install_dependencies_on_cluster.sh \
-		--metadata="PACKAGE=$(BUCKET_NAME)/${APP_NAME}-${PACKAGE_VERSION}-py3-none-any.whl" \
+		--master-machine-type n1-standard-2 \
+		--metadata="GENTROPY_REF=${REF}" \
+		--initialization-actions=${BUCKET_NAME}/install_dependencies_on_cluster.sh \
 		--secondary-worker-type spot \
 		--worker-machine-type n1-standard-4 \
+		--public-ip-address \
 		--worker-boot-disk-size 500 \
 		--autoscaling-policy="projects/${PROJECT_ID}/regions/${REGION}/autoscalingPolicies/otg-etl" \
 		--optional-components=JUPYTER \
 		--enable-component-gateway \
 		--max-idle=60m
 
-make update-dev-cluster: build ## Reinstalls the package on the dev-cluster
+update-dev-cluster: build ## Reinstalls the package on the dev-cluster
 	@echo "Updating Dataproc Dev Cluster"
 	@gcloud config set project ${PROJECT_ID}
 	gcloud dataproc jobs submit pig --cluster="ot-genetics-dev-${CLEAN_PACKAGE_VERSION}" \
@@ -68,10 +80,4 @@ make update-dev-cluster: build ## Reinstalls the package on the dev-cluster
 		-e='sh chmod 750 $${PWD}/install_dependencies_on_cluster.sh; sh $${PWD}/install_dependencies_on_cluster.sh'
 
 build: clean ## Build Python package with dependencies
-	@gcloud config set project ${PROJECT_ID}
-	@echo "Packaging Code and Dependencies for ${APP_NAME}-${PACKAGE_VERSION}"
-	@poetry build
-	@echo "Uploading to ${BUCKET_NAME}"
-	@gsutil cp src/${APP_NAME}/cli.py ${BUCKET_NAME}/
-	@gsutil cp ./dist/${APP_NAME}-${PACKAGE_VERSION}-py3-none-any.whl ${BUCKET_NAME}/
-	@gsutil cp ./utils/install_dependencies_on_cluster.sh ${BUCKET_NAME}/
+	@uv build

docs/development/contributing.md

@@ -18,27 +18,29 @@ For Google Cloud configuration:
 
 Check that you have the `make` utility installed, and if not (which is unlikely), install it using your system package manager.
 
-Check that you have `java` installed. To be able to use all features including hail support use java 11.
+!!! note "Java support"
+
+    Check that you have `java` installed. To be able to use all features including hail support use java 11 (for handling multiple java versions, consider using [`sdkman`](https://sdkman.io/)).
 
 ## Environment configuration
 
-Run `make setup-dev` to install/update the necessary packages and activate the development environment. You need to do this every time you open a new shell.
+Run `make setup-dev` to install/update the necessary packages (including required python version for development) and activate the development environment. You need to do it just once.
 
 It is recommended to use VS Code as an IDE for development.
 
 ## How to create gentropy step
 
 All gentropy steps can be invoked after successful environment configuration by running
 
-```python
-poetry run gentropy step=<step_name>
+```bash
+uv run gentropy step=<step_name>
 ```
 
 1. Create a new step config in the `src/gentropy/config.py` that inherits from `StepConfig` class.
 
 2. Register new step configuration to `ConfigStore`.
 
-3. Create a step class that holds the business logic in new file in the `src/gentropy`.
+3. Create a step class that holds the business logic in new file in the `src/gentropy/{your_step_name}.py`.
 
 ## Contributing checklist
 
@@ -78,3 +80,7 @@ For more details on each of these steps, see the sections below.
 ### Airflow dags
 
 - Upstream of version 2.0.0 airflow orchestration layer was moved to the [orchestration repository](https://github.com/opentargets/orchestration)
+
+### Support for python versions
+
+As of version 2.1.X gentropy supports multiple python versions. To ensure compatibility with all supported versions, unit tests are run for each of the minor python release from 3.10 to 3.12. Make sure your changes are compatible with all supported versions.