Merge pull request #858 from opentargets/dev

chore: trigger release process

.github/workflows/artifact.yml

@@ -2,14 +2,15 @@ name: Build and Push to Artifact Registry
 
 "on":
   push:
-    branches: ["dev"]
-    tags: ["v*"]
+    branches: ["*"]
+    tags: ["*"]
 
 env:
   PROJECT_ID: open-targets-genetics-dev
   REGION: europe-west1
   GAR_LOCATION: europe-west1-docker.pkg.dev/open-targets-genetics-dev
   REPOSITORY: gentropy-app
+  PYTHON_VERSION_DEFAULT: "3.10.8"
 
 jobs:
   build-push-artifact:
@@ -39,7 +40,18 @@ jobs:
         run: |-
           gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet
 
+      - name: Quick Docker build (gentropy only, AMD64 only, with layer cache)
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64
+          push: true
+          tags: "${{ env.GAR_LOCATION }}/${{ env.REPOSITORY }}/gentropy:${{ github.ref_name }}"
+          context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build and push gentropy image
+        if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
@@ -48,10 +60,26 @@ jobs:
           context: .
 
       - name: Build and push VEP image
+        if: github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/v')
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64
           push: true
           tags: "${{ env.GAR_LOCATION }}/${{ env.REPOSITORY }}/custom_ensembl_vep:${{ github.ref_name }}"
           context: .
           file: "src/vep/Dockerfile"
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION_DEFAULT }}
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Build and push spark cluster dependencies
+        run: |
+          make build

.github/workflows/pr.yaml

@@ -17,6 +17,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.10.8
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: "8"
+          distribution: "temurin"
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:
@@ -41,7 +46,7 @@ jobs:
       - name: Run tests
         run: poetry run pytest
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml

.gitignore

@@ -13,3 +13,4 @@ site/
 .env
 .coverage*
 wandb/
+hail*.log

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ ci:
   skip: [poetry-lock]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.2
+    rev: v0.7.4
     hooks:
       - id: ruff
         args:
@@ -15,7 +15,7 @@ repos:
         files: ^((gentropy|utils|tests)/.+)?[^/]+\.py$
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -58,14 +58,14 @@ repos:
         exclude: "CHANGELOG.md"
 
   - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
-    rev: v9.16.0
+    rev: v9.18.0
     hooks:
       - id: commitlint
         additional_dependencies: ["@commitlint/config-conventional@18.6.3"]
         stages: [commit-msg]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.1"
+    rev: "v1.13.0"
     hooks:
       - id: mypy
         args:
@@ -98,7 +98,7 @@ repos:
       - id: beautysh
 
   - repo: https://github.com/jsh9/pydoclint
-    rev: 0.5.4
+    rev: 0.5.9
     hooks:
       - id: pydoclint
 

Makefile

@@ -1,10 +1,17 @@
 PROJECT_ID ?= open-targets-genetics-dev
 REGION ?= europe-west1
-APP_NAME ?= $$(cat pyproject.toml| grep -m 1 "name" | cut -d" " -f3 | sed  's/"//g')
-VERSION_NO ?= $$(poetry version --short)
-CLEAN_VERSION_NO := $(shell echo "$(VERSION_NO)" | tr -cd '[:alnum:]')
-BUCKET_NAME=gs://genetics_etl_python_playground/initialisation/${VERSION_NO}/
-BUCKET_COMPOSER_DAGS=gs://europe-west1-ot-workflows-fe147745-bucket/dags/
+APP_NAME ?= $$(cat pyproject.toml | grep -m 1 "name" | cut -d" " -f3 | sed  's/"//g')
+PACKAGE_VERSION ?= $$(poetry version --short)
+# NOTE: git rev-parse will always return the HEAD if it sits in the tag,
+# this way we can distinguish the tag vs branch name
+ifeq ($(shell git rev-parse --abbrev-ref HEAD),HEAD)
+	REF := $(shell git describe --exact-match --tags)
+else
+	REF := $(shell git rev-parse --abbrev-ref HEAD)
+endif
+
+CLEAN_PACKAGE_VERSION := $(shell echo "$(PACKAGE_VERSION)" | tr -cd '[:alnum:]')
+BUCKET_NAME=gs://genetics_etl_python_playground/initialisation/${APP_NAME}/${REF}
 
 .PHONY: $(shell sed -n -e '/^$$/ { n ; /^[^ .\#][^ ]*:/ { s/:.*$$// ; p ; } ; }' $(MAKEFILE_LIST))
 
@@ -38,34 +45,33 @@ build-documentation: ## Create local server with documentation
 create-dev-cluster: build ## Spin up a simple dataproc cluster with all dependencies for development purposes
 	@echo "Creating Dataproc Dev Cluster"
 	@gcloud config set project ${PROJECT_ID}
-	@gcloud dataproc clusters create "ot-genetics-dev-${CLEAN_VERSION_NO}" \
+	@gcloud dataproc clusters create "ot-genetics-dev-${CLEAN_PACKAGE_VERSION}-$(USER)" \
 		--image-version 2.1 \
 		--region ${REGION} \
 		--master-machine-type n1-standard-16 \
-		--initialization-actions=gs://genetics_etl_python_playground/initialisation/${VERSION_NO}/install_dependencies_on_cluster.sh \
-		--metadata="PACKAGE=gs://genetics_etl_python_playground/initialisation/${VERSION_NO}/gentropy-${VERSION_NO}-py3-none-any.whl,CONFIGTAR=gs://genetics_etl_python_playground/initialisation/${VERSION_NO}/config.tar.gz" \
-		--primary-worker-type n1-standard-8 \
+		--initialization-actions=$(BUCKET_NAME)/install_dependencies_on_cluster.sh \
+		--metadata="PACKAGE=$(BUCKET_NAME)/${APP_NAME}-${PACKAGE_VERSION}-py3-none-any.whl" \
+		--secondary-worker-type spot \
 		--worker-machine-type n1-standard-4 \
 		--worker-boot-disk-size 500 \
-		--autoscaling_policy=f"projects/${PROJECT_ID}/regions/${REGION}/autoscalingPolicies/eqtl-preprocess", \
+		--autoscaling-policy="projects/${PROJECT_ID}/regions/${REGION}/autoscalingPolicies/otg-etl" \
 		--optional-components=JUPYTER \
-		--enable-component-gateway
+		--enable-component-gateway \
+		--max-idle=60m
 
 make update-dev-cluster: build ## Reinstalls the package on the dev-cluster
 	@echo "Updating Dataproc Dev Cluster"
 	@gcloud config set project ${PROJECT_ID}
-	gcloud dataproc jobs submit pig --cluster="ot-genetics-dev-${CLEAN_VERSION_NO}" \
+	gcloud dataproc jobs submit pig --cluster="ot-genetics-dev-${CLEAN_PACKAGE_VERSION}" \
 		--region ${REGION} \
 		--jars=${BUCKET_NAME}/install_dependencies_on_cluster.sh \
 		-e='sh chmod 750 $${PWD}/install_dependencies_on_cluster.sh; sh $${PWD}/install_dependencies_on_cluster.sh'
 
 build: clean ## Build Python package with dependencies
 	@gcloud config set project ${PROJECT_ID}
-	@echo "Packaging Code and Dependencies for ${APP_NAME}-${VERSION_NO}"
+	@echo "Packaging Code and Dependencies for ${APP_NAME}-${PACKAGE_VERSION}"
 	@poetry build
-	@tar -czf dist/config.tar.gz config/
-	@echo "Uploading to Dataproc"
-	@gsutil cp src/gentropy/cli.py ${BUCKET_NAME}
-	@gsutil cp ./dist/${APP_NAME}-${VERSION_NO}-py3-none-any.whl ${BUCKET_NAME}
-	@gsutil cp ./dist/config.tar.gz ${BUCKET_NAME}
-	@gsutil cp ./utils/install_dependencies_on_cluster.sh ${BUCKET_NAME}
+	@echo "Uploading to ${BUCKET_NAME}"
+	@gsutil cp src/${APP_NAME}/cli.py ${BUCKET_NAME}/
+	@gsutil cp ./dist/${APP_NAME}-${PACKAGE_VERSION}-py3-none-any.whl ${BUCKET_NAME}/
+	@gsutil cp ./utils/install_dependencies_on_cluster.sh ${BUCKET_NAME}/