From 8c98b46b40e5779eb789b3b510d0352dc45a34f0 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 09:12:09 +0100 Subject: [PATCH 01/10] ci: ci test release --- .github/workflows/pr.yaml | 2 +- .github/workflows/release.yaml | 39 +++++++++ .github/workflows/release_pr.yaml | 34 ++++++++ .github/workflows/tag.yaml | 139 ++++++++++++++++++++++++++++++ pyproject.toml | 67 +++++++++++++- 5 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/release_pr.yaml create mode 100644 .github/workflows/tag.yaml diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 10b99d8..2f09a10 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,4 +20,4 @@ jobs: - name: Install the latest version of rye run: make dev - name: Run checks - - uses: pre-commit/action@v3.0.1 + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..346efef --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,39 @@ +name: Release + +on: + push: + tags: + - "v*" + +env: + PYTHON_VERSION_DEFAULT: "3.10.8" + +jobs: + release: + runs-on: ubuntu-latest + environment: PyPI + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + contents: write + steps: + - uses: actions/create-github-app-token@v1 + id: trigger-token + with: + app-id: ${{ vars.TRIGGER_WORKFLOW_GH_APP_ID}} + private-key: ${{ secrets.TRIGGER_WORKFLOW_GH_APP_KEY }} + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.8 + - name: Install the latest version of rye + run: make dev + - name: Build package + run: make build + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: dist/ diff --git a/.github/workflows/release_pr.yaml b/.github/workflows/release_pr.yaml new file mode 100644 index 0000000..441d43a --- /dev/null +++ b/.github/workflows/release_pr.yaml @@ -0,0 +1,34 @@ +name: PR to trigger release + +"on": + schedule: + - cron: "15 16 * * 4" + +jobs: + pull-request: + runs-on: ubuntu-latest + environment: PyPI + steps: + - uses: actions/create-github-app-token@v1 + id: trigger-token + with: + app-id: ${{ vars.TRIGGER_WORKFLOW_GH_APP_ID}} + private-key: ${{ secrets.TRIGGER_WORKFLOW_GH_APP_KEY }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: opentargets/gentropy + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: false + - uses: actions/checkout@v3 + - name: pull-request + uses: diillson/auto-pull-request@v1.0.1 + with: + source_branch: "dev" + destination_branch: "main" + pr_title: "chore: trigger release process" + pr_body: ":warning: *This PR requires a MERGE COMMIT (Don't squash or rebase!)*" + pr_label: "auto-pr" + pr_draft: false + pr_allow_empty: true + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml new file mode 100644 index 0000000..1722468 --- /dev/null +++ b/.github/workflows/tag.yaml @@ -0,0 +1,139 @@ +name: Release + +"on": + push: + branches: ["main"] + +concurrency: + group: deploy + cancel-in-progress: false # prevent hickups with semantic-release + +env: + PYTHON_VERSION_DEFAULT: "3.10.8" + +jobs: + release: + runs-on: ubuntu-latest + concurrency: release + environment: PyPI + permissions: + contents: write + + steps: + # NOTE: commits using GITHUB_TOKEN does not trigger workflows + - uses: actions/create-github-app-token@v1 + id: trigger-token + with: + app-id: ${{ vars.TRIGGER_WORKFLOW_GH_APP_ID}} + private-key: ${{ secrets.TRIGGER_WORKFLOW_GH_APP_KEY }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: opentargets/gentropy + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: false + + - name: Python Semantic Release + id: semrelease + # v9.6.0 is required due to the python v3.12 in the newer version of semantic release action which + # breaks the poetry build command. + uses: python-semantic-release/python-semantic-release@v9.6.0 + with: + github_token: ${{ steps.trigger-token.outputs.token }} + + - name: Publish package to GitHub Release + uses: python-semantic-release/upload-to-gh-release@main + if: ${{ steps.semrelease.outputs.released }} == 'true' + with: + # NOTE: allow to start the workflow when push action on tag gets executed + # requires using GH_APP to authenitcate, otherwise push authorised with + # the GITHUB_TOKEN does not trigger the tag artifact workflow. + # see https://github.com/actions/create-github-app-token + github_token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ steps.semrelease.outputs.tag }} + + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + needs: release + name: >- + Publish 📦 in PyPI + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/gentropy + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish 📦 in TestPyPI + needs: release + if: github.ref != 'refs/heads/main' + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/gentropy + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + documentation: + needs: release + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION_DEFAULT }} + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: | + venv-${{ runner.os }}-\ + ${{ env.PYTHON_VERSION_DEFAULT }}-\ + ${{ hashFiles('**/poetry.lock') }} + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + - name: Install library + run: poetry install --without tests --no-interaction + - name: Publish docs + run: poetry run mkdocs gh-deploy --force diff --git a/pyproject.toml b/pyproject.toml index 0e7f05b..abe20a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "gentroutils" -version = "0.1.0" +version = "0.0.0" description = "Add your description here" authors = [ { name = "Szymon Szyszkowski", email = "ss60@mib117351s.internal.sanger.ac.uk" }, @@ -126,3 +126,68 @@ warn_unused_ignores = true [[tool.mypy.overrides]] module = ["google.cloud", "pyfiglet", "click", "google.cloud.storage"] ignore_missing_imports = true + + +[tool.semantic_release] +version_toml = ["pyproject.tolm:project.version"] +assets = [] +commit_message = "{version}\n\nAutomatically generated by python-semantic-release" +commit_parser = "angular" +logging_use_named_masks = false +major_on_zero = true +allow_zero_version = true +tag_format = "v{version}" + +[tool.semantic_release.branches.main] +match = "(main|master)" +prerelease_token = "rc" +prerelease = false + +[semantic_release.changelog] +template_dir = "templates" +changelog_file = "CHANGELOG.md" +exclude_commit_patterns = [] + +[tool.semantic_release.changelog.environment] +block_start_string = "{%" +block_end_string = "%}" +variable_start_string = "{{" +variable_end_string = "}}" +comment_start_string = "{#" +comment_end_string = "#}" +trim_blocks = false +lstrip_blocks = false +newline_sequence = "\n" +keep_trailing_newline = false +extensions = [] +autoescape = true + +[tool.semantic_release.commit_author] +env = "GIT_COMMIT_AUTHOR" +default = "semantic-release " + +[tool.semantic_release.commit_parser_options] +allowed_tags = [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "style", + "refactor", + "test", +] +minor_tags = ["feat"] +patch_tags = ["fix", "perf", "docs", "style", "test", "ci"] +default_bump_level = 0 + +[tool.semantic_release.remote] +name = "origin" +type = "github" +ignore_token_for_push = false + +[tool.semantic_release.publish] +dist_glob_patterns = ["dist/*"] +upload_to_vcs_release = true From 7e1340f1fb4aa57092f149db0e9400d18a4a0cca Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 09:19:35 +0100 Subject: [PATCH 02/10] fix: add cargo to rc file --- setup.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.sh b/setup.sh index e77988b..c02e690 100755 --- a/setup.sh +++ b/setup.sh @@ -5,24 +5,25 @@ # It is intended to be run on a new system to quickly set up Rye. # It is recommended to review the script before running it. +export SHELL_RC=$(echo "$HOME/.${SHELL##*/}rc") -export SHELL_ZSH=$(echo "$HOME/.${SHELL##*/}rc") - -if ! command -v rye &> /dev/null; then +if ! command -v rye &>/dev/null; then echo "Rye is not installed. Installing..." curl -sSf https://rye.astral.sh/get | bash - echo "Updating $SHELL_ZSH" - echo "source $HOME/.rye/env" >> $SHELL_ZSH + echo "Updating $SHELL_RC" + echo "source $HOME/.rye/env" >>$SHELL_RC + echo "source $HOME/.cargo/bin" >>$SHELL_RC else echo "Rye is already installed." fi -if ! command -v uv &> /dev/null; then +if ! command -v uv &>/dev/null; then echo "uv is not installed. Installing..." curl -LsSf https://astral.sh/uv/install.sh | sh else echo "uv is already installed." fi +source $SHELL_RC rye sync rye run pre-commit install --hook-type commit-msg --hook-type pre-commit From cd0c7eda865af9df5bf1ab65dfa405b3a13e605c Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 09:28:47 +0100 Subject: [PATCH 03/10] fix: debug ci --- setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.sh b/setup.sh index c02e690..b8c451b 100755 --- a/setup.sh +++ b/setup.sh @@ -24,6 +24,7 @@ else echo "uv is already installed." fi source $SHELL_RC +cat $SHELL_RC rye sync rye run pre-commit install --hook-type commit-msg --hook-type pre-commit From 332e0d58dee315c9aa5f57f20464d1097c41cb59 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:12:03 +0100 Subject: [PATCH 04/10] fix: debug v2 --- .github/workflows/tag.yaml | 86 -------------------------------------- setup.sh | 4 +- 2 files changed, 3 insertions(+), 87 deletions(-) diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml index 1722468..0997101 100644 --- a/.github/workflows/tag.yaml +++ b/.github/workflows/tag.yaml @@ -51,89 +51,3 @@ jobs: # see https://github.com/actions/create-github-app-token github_token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ steps.semrelease.outputs.tag }} - - - name: Store the distribution packages - uses: actions/upload-artifact@v4 - with: - name: python-package-distributions - path: dist/ - - publish-to-pypi: - needs: release - name: >- - Publish 📦 in PyPI - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/gentropy - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - name: python-package-distributions - path: dist/ - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - publish-to-testpypi: - name: Publish 📦 in TestPyPI - needs: release - if: github.ref != 'refs/heads/main' - runs-on: ubuntu-latest - - environment: - name: testpypi - url: https://test.pypi.org/p/gentropy - - permissions: - id-token: write # IMPORTANT: mandatory for trusted publishing - - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - name: python-package-distributions - path: dist/ - - name: Publish distribution 📦 to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - - documentation: - needs: release - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION_DEFAULT }} - - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v4 - with: - path: .venv - key: | - venv-${{ runner.os }}-\ - ${{ env.PYTHON_VERSION_DEFAULT }}-\ - ${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --without tests --no-interaction - - name: Publish docs - run: poetry run mkdocs gh-deploy --force diff --git a/setup.sh b/setup.sh index b8c451b..8c7883e 100755 --- a/setup.sh +++ b/setup.sh @@ -24,7 +24,9 @@ else echo "uv is already installed." fi source $SHELL_RC -cat $SHELL_RC +which rye +echo $PATH +cat $HOME/.rye/env rye sync rye run pre-commit install --hook-type commit-msg --hook-type pre-commit From ce4fd37ed2483c353bc552e6d9e91b5f9883dc52 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:29:07 +0100 Subject: [PATCH 05/10] feat: attempt to install rye with cargo --- setup.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index 8c7883e..9460f0f 100755 --- a/setup.sh +++ b/setup.sh @@ -7,12 +7,21 @@ export SHELL_RC=$(echo "$HOME/.${SHELL##*/}rc") +if ! command -v cargo &>/dev/null; then + echo "Cargo is not installed. Installing..." + curl https://sh.rustup.rs -sSf | sh + echo "Updating $SHELL_RC" + echo "source $HOME/.cargo/bin" >>$SHELL_RC +else + echo "Cargo is installed." +fi +source $SHELL_RC + if ! command -v rye &>/dev/null; then echo "Rye is not installed. Installing..." - curl -sSf https://rye.astral.sh/get | bash + cargo install --git https://github.com/astral-sh/rye --tag 0.38.0 rye echo "Updating $SHELL_RC" echo "source $HOME/.rye/env" >>$SHELL_RC - echo "source $HOME/.cargo/bin" >>$SHELL_RC else echo "Rye is already installed." fi From e7c9e5e2dec910aa9cef2b14b1d88d4e669c057a Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:39:32 +0100 Subject: [PATCH 06/10] ci: changed how uv and rye are installed --- .github/workflows/pr.yaml | 10 ++++++++-- .github/workflows/release.yaml | 10 ++++++++-- setup.sh | 16 ++-------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2f09a10..178a69f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,13 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.10.8 - - name: Install the latest version of rye - run: make dev + - name: Install uv + uses: yezz123/setup-uv@v4 + - name: Install rye + uses: eifinger/setup-rye@v4 + with: + version: "latest" + - name: Sync dependencies + run: rye sync - name: Run checks uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 346efef..bf783ff 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,8 +28,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.10.8 - - name: Install the latest version of rye - run: make dev + - name: Install uv + uses: yezz123/setup-uv@v4 + - name: Install rye + uses: eifinger/setup-rye@v4 + with: + version: "latest" + - name: Sync dependencies + run: rye sync - name: Build package run: make build - name: Publish distribution 📦 to TestPyPI diff --git a/setup.sh b/setup.sh index 9460f0f..c02e690 100755 --- a/setup.sh +++ b/setup.sh @@ -7,21 +7,12 @@ export SHELL_RC=$(echo "$HOME/.${SHELL##*/}rc") -if ! command -v cargo &>/dev/null; then - echo "Cargo is not installed. Installing..." - curl https://sh.rustup.rs -sSf | sh - echo "Updating $SHELL_RC" - echo "source $HOME/.cargo/bin" >>$SHELL_RC -else - echo "Cargo is installed." -fi -source $SHELL_RC - if ! command -v rye &>/dev/null; then echo "Rye is not installed. Installing..." - cargo install --git https://github.com/astral-sh/rye --tag 0.38.0 rye + curl -sSf https://rye.astral.sh/get | bash echo "Updating $SHELL_RC" echo "source $HOME/.rye/env" >>$SHELL_RC + echo "source $HOME/.cargo/bin" >>$SHELL_RC else echo "Rye is already installed." fi @@ -33,9 +24,6 @@ else echo "uv is already installed." fi source $SHELL_RC -which rye -echo $PATH -cat $HOME/.rye/env rye sync rye run pre-commit install --hook-type commit-msg --hook-type pre-commit From 7d74378a45c307b9ae64cd6dfd707948113b5e7d Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:45:04 +0100 Subject: [PATCH 07/10] fix: ensure pip is installed for pre-commit action --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 178a69f..eeebb5a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,5 +25,7 @@ jobs: version: "latest" - name: Sync dependencies run: rye sync + - name: ensure pip + run: python -m ensurepip # required as pre-commit requires pip to install - name: Run checks uses: pre-commit/action@v3.0.1 From 4345691c7e3a352b72d5003fbfe5781d7d111c90 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:52:05 +0100 Subject: [PATCH 08/10] feat: local pre-commit execution without action --- .github/workflows/pr.yaml | 4 +--- Makefile | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eeebb5a..4cf20ae 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,7 +25,5 @@ jobs: version: "latest" - name: Sync dependencies run: rye sync - - name: ensure pip - run: python -m ensurepip # required as pre-commit requires pip to install - name: Run checks - uses: pre-commit/action@v3.0.1 + run: pre-commit run --all-files diff --git a/Makefile b/Makefile index 4717f5a..a9f49d0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash .PHONY: $(shell sed -n -e '/^$$/ { n ; /^[^ .\#][^ ]*:/ { s/:.*$$// ; p ; } ; }' $(MAKEFILE_LIST)) VERSION := $$(grep '^version' pyproject.toml | sed 's%version = "\(.*\)"%\1%') -APP_NAME := $$(grep '^name' pyproject.toml | sed 's%name = "\(.*\)"%\1%') +APP_NAME := $$(grep '^name' pyproject.toml | head -1 | sed 's%name = "\(.*\)"%\1%') .DEFAULT_GOAL := help From 10584a56790cf4693bd47ee3e50672bc4bb78068 Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:54:06 +0100 Subject: [PATCH 09/10] fix: typo in package name --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bf783ff..f6b461c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,5 +41,5 @@ jobs: - name: Publish distribution 📦 to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - repository-url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/gentroutils/ packages-dir: dist/ From 7fbd06cbeb42563f6c2a66f768e490f554a7c07b Mon Sep 17 00:00:00 2001 From: Szymon Szyszkowski Date: Tue, 13 Aug 2024 10:55:10 +0100 Subject: [PATCH 10/10] fix: run pre-commit by rye --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4cf20ae..bf8ce5d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -26,4 +26,4 @@ jobs: - name: Sync dependencies run: rye sync - name: Run checks - run: pre-commit run --all-files + run: rye run pre-commit run --all-files