From fef08746fd8cc393fd41636e2b57e82d05916eb3 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 22 Oct 2024 10:46:17 +0200 Subject: [PATCH] ci: Use the release-metadata action (#597) - closes #598 --------- Co-authored-by: Apify Release Bot --- .github/workflows/_publish_to_pypi.yaml | 70 ---------- .github/workflows/pre_release.yaml | 71 ++++++++++ .github/workflows/release.yaml | 102 ++++++++++++++ .github/workflows/run_release.yaml | 175 ------------------------ CHANGELOG.md | 2 +- Makefile | 2 +- cliff.toml | 102 -------------- scripts/fetch_pr_issues.sh | 32 ----- scripts/preprocess_commit_message.py | 54 -------- 9 files changed, 175 insertions(+), 435 deletions(-) delete mode 100644 .github/workflows/_publish_to_pypi.yaml create mode 100644 .github/workflows/pre_release.yaml create mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/run_release.yaml delete mode 100644 cliff.toml delete mode 100755 scripts/fetch_pr_issues.sh delete mode 100644 scripts/preprocess_commit_message.py diff --git a/.github/workflows/_publish_to_pypi.yaml b/.github/workflows/_publish_to_pypi.yaml deleted file mode 100644 index 323bc55f9..000000000 --- a/.github/workflows/_publish_to_pypi.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: Publish to PyPI - -on: - workflow_call: - inputs: - version_number: - required: true - type: string - -env: - PYTHON_VERSION: 3.12 - -jobs: - publish_to_pypi: - name: Publish to PyPI - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write # Required for OIDC authentication. - environment: - name: pypi - url: https://pypi.org/project/crawlee - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install dependencies - run: | - pipx install --python ${{ env.PYTHON_VERSION }} poetry - make install-dev - - # Updates the version number in the project's configuration. - - name: Set version in pyproject.toml - run: | - if [[ ${{ github.event_name }} = push ]]; then - # Make a pre-release on push - next_beta=$( - curl "https://pypi.org/pypi/crawlee/json" | jq --raw-output ' - [ - .releases | - keys | - .[] | - select(startswith("${{ inputs.version_number }}b")) | - split("b") | - .[1] | - tonumber - ] | - sort_by(- .) | - .[0] + 1 - ' - ) - poetry version "${{ inputs.version_number }}b$next_beta" - else - # ...and a regular release otherwise - poetry version '${{ inputs.version_number }}' - fi - - # Builds the package. - - name: Build package - run: make build - - # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/pre_release.yaml b/.github/workflows/pre_release.yaml new file mode 100644 index 000000000..5f1d67a48 --- /dev/null +++ b/.github/workflows/pre_release.yaml @@ -0,0 +1,71 @@ +name: Create a pre-release + +on: + # Trigger a beta version release (pre-release) on push to the master branch. + push: + branches: + - master + tags-ignore: + - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. + +jobs: + release_metadata: + name: Prepare release metadata + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + steps: + - uses: apify/workflows/git-cliff-release@main + id: release_metadata + name: Prepare release metadata + with: + release_type: prerelease + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/crawlee + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: crawlee + is_prerelease: "yes" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # TODO: add job for publish package to Conda + # https://github.com/apify/crawlee-python/issues/104 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..6a3afb227 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,102 @@ +name: Create a release + +on: + # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release. + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + type: choice + default: auto + options: + - auto + - custom + - patch + - minor + - major + custom_version: + description: The custom version to bump to (only for "custom" type) + required: false + type: string + default: "" + +jobs: + release_metadata: + if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci')" + name: Prepare release metadata + runs-on: ubuntu-latest + outputs: + version_number: ${{ steps.release_metadata.outputs.version_number }} + tag_name: ${{ steps.release_metadata.outputs.tag_name }} + changelog: ${{ steps.release_metadata.outputs.changelog }} + release_notes:: ${{ steps.release_metadata.outputs.release_notes }} + steps: + - uses: apify/workflows/git-cliff-release@main + name: Prepare release metadata + id: release_metadata + with: + release_type: ${{ inputs.release_type }} + custom_version: ${{ inputs.custom_version }} + + lint_check: + name: Lint check + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + + type_check: + name: Type check + uses: apify/workflows/.github/workflows/python_type_check.yaml@main + + unit_tests: + name: Unit tests + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main + + update_changelog: + name: Update changelog + needs: [release_metadata, lint_check, type_check, unit_tests] + uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main + with: + version_number: ${{ needs.release_metadata.outputs.version_number }} + changelog: ${{ needs.release_metadata.outputs.changelog }} + secrets: + APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + create_github_release: + name: Create github release + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Create release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.release_metadata.outputs.tag_name }} + name: ${{ needs.release_metadata.outputs.version_number }} + target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }} + body: ${{ needs.release_metadata.outputs.release_notes }} + + publish_to_pypi: + name: Publish to PyPI + needs: [release_metadata, update_changelog] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # Required for OIDC authentication. + environment: + name: pypi + url: https://pypi.org/project/crawlee + steps: + - name: Prepare distribution + uses: apify/workflows/prepare-pypi-distribution@main + with: + package_name: crawlee + is_prerelease: "" + version_number: ${{ needs.release_metadata.outputs.version_number }} + ref: ${{ needs.update_changelog.changelog_commitish }} + # Publishes the package to PyPI using PyPA official GitHub action with OIDC authentication. + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # TODO: add job for publish package to Conda + # https://github.com/apify/crawlee-python/issues/104 diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml deleted file mode 100644 index 154616e1a..000000000 --- a/.github/workflows/run_release.yaml +++ /dev/null @@ -1,175 +0,0 @@ -name: Run release - -on: - # Trigger a beta version release (pre-release) on push to the master branch. - push: - branches: - - master - tags-ignore: - - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. - - # Trigger a stable version release via GitHub's UI, with the ability to specify the type of release. - workflow_dispatch: - inputs: - release_type: - description: Release type - required: true - type: choice - default: auto - options: - - auto - - custom - custom_version: - description: The custom version to bump to (only for "custom" type) - required: false - type: string - default: "" - -jobs: - # This job determines if the conditions are met for a release to occur. It will proceed if triggered manually, - # for any published release, or if the commit on push does not begin with "docs" or "chore". - should_release: - name: Check whether to release - if: | - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'push' && - !startsWith(github.event.head_commit.message, 'docs') && - !startsWith(github.event.head_commit.message, 'ci') - ) - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - outputs: - version_number: ${{ steps.version_number.outputs.result }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Locally remove beta tags - run: | - git tag | grep 'b[0-9]' | xargs git tag --delete - - - name: Install git-cliff - run: pip install git-cliff - - - name: Determine version number - id: version_number - run: | - if [[ '${{ github.event_name }}' = workflow_dispatch && '${{ github.event.inputs.release_type }}' = custom ]]; then - echo result=$(echo ${{ github.event.inputs.custom_version }} | sed s/^v//) | tee -a $GITHUB_OUTPUT - else - echo result=$(git-cliff --bumped-version | sed s/^v//) | tee -a $GITHUB_OUTPUT - fi - - lint_check: - name: Lint check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main - - type_check: - name: Type check - needs: [should_release] - uses: apify/workflows/.github/workflows/python_type_check.yaml@main - - unit_tests: - name: Unit tests - needs: [should_release] - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - - update_changelog: - name: Update changelog - needs: [should_release, lint_check, type_check, unit_tests] - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write # Required for OIDC authentication. - outputs: - changelog_commitish: ${{ steps.commit.commit_long_sha || github.sha }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYTHON_VERSION: 3.12 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Locally remove beta tags - run: | - git tag | grep 'b[0-9]' | xargs git tag --delete - - - name: Install git-cliff - run: pip install git-cliff - - - name: Generate changelog with git-cliff - run: | - if [[ ${{ github.event_name }} = workflow_dispatch ]]; then - git-cliff --tag v${{ needs.should_release.outputs.version_number }} > CHANGELOG.md - else - git-cliff --with-tag-message v${{ needs.should_release.outputs.version_number }} > CHANGELOG.md - fi - - - name: Install poetry - run: pipx install --python ${{ env.PYTHON_VERSION }} poetry - - - name: Update package version in pyproject.toml - run: poetry version ${{ needs.should_release.outputs.version_number }} - - - name: Commit changes - id: commit - uses: EndBug/add-and-commit@v9 - with: - author_name: Apify Release Bot - author_email: noreply@apify.com - message: "chore(release): Update changelog and package version [skip ci]" - - create_github_release: - name: Create github release - if: github.event_name == 'workflow_dispatch' - needs: [should_release, lint_check, type_check, unit_tests, update_changelog] - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Locally remove beta tags - run: | - git tag | grep 'b[0-9]' | xargs git tag --delete - - - name: Install git-cliff - run: pip install git-cliff - - - name: Generate release notes with git-cliff - run: git-cliff --tag "v${{ needs.should_release.outputs.version_number }}" --unreleased --strip all > release_notes.md - - - name: Create release - uses: softprops/action-gh-release@v2 - with: - tag_name: v${{ needs.should_release.outputs.version_number }} - name: ${{ needs.should_release.outputs.version_number }} - target_commitish: ${{ needs.update_changelog.outputs.changelog_commitish }} - body_path: release_notes.md - - publish_to_pypi: - name: Publish to PyPI - needs: [should_release, lint_check, type_check, unit_tests] - uses: ./.github/workflows/_publish_to_pypi.yaml - with: - version_number: ${{ needs.should_release.outputs.version_number }} - - # TODO: add job for publish package to Conda - # https://github.com/apify/crawlee-python/issues/104 diff --git a/CHANGELOG.md b/CHANGELOG.md index f1dbc9e0d..34a18ff47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -230,4 +230,4 @@ All notable changes to this project will be documented in this file. - Fire local SystemInfo events every second ([#144](https://github.com/apify/crawlee-python/pull/144)) ([f1359fa](https://github.com/apify/crawlee-python/commit/f1359fa7eea23f8153ad711287c073e45d498401)) by [@vdusek](https://github.com/vdusek) - Storage manager & purging the defaults ([#150](https://github.com/apify/crawlee-python/pull/150)) ([851042f](https://github.com/apify/crawlee-python/commit/851042f25ad07e25651768e476f098ef0ed21914)) by [@vdusek](https://github.com/vdusek) - + \ No newline at end of file diff --git a/Makefile b/Makefile index bf561030e..6649f382d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests format check-code build-api-reference run-doc -DIRS_WITH_CODE = src tests scripts docs +DIRS_WITH_CODE = src tests docs # This is default for local testing, but GitHub workflows override it to a higher value in CI INTEGRATION_TESTS_CONCURRENCY = 1 diff --git a/cliff.toml b/cliff.toml deleted file mode 100644 index 390270a52..000000000 --- a/cliff.toml +++ /dev/null @@ -1,102 +0,0 @@ -# git-cliff ~ default configuration file -# https://git-cliff.org/docs/configuration -# -# Lines starting with "#" are comments. -# Configuration options are organized into tables and keys. -# See documentation for more information on available options. - -[changelog] -# changelog header -header = """ -# Changelog\n -All notable changes to this project will be documented in this file.\n -""" -# template for the changelog body -# https://keats.github.io/tera/docs/#introduction -body = """ -{% if version %}\ - ## [{{ version | trim_start_matches(pat="v") }}](/releases/tag/{{ version }}) ({{ timestamp | date(format="%Y-%m-%d") }}) -{% elif message %}\ - ## {{ message | trim_start_matches(pat="v") }} - **not yet released** -{% else %}\ - ## unreleased -{% endif %}\ -{% for group, commits in commits | group_by(attribute="group") %} - ### {{ group | striptags | trim | upper_first }} - {% for commit in commits %} - - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ - {% if commit.breaking %}[**breaking**] {% endif %}\ - {{ commit.message | upper_first }} ([{{ commit.id | truncate(length = 7, end = "") }}](/commit/{{ commit.id }}))\ - {% if commit.github.username %} by [@{{ commit.github.username }}](https://github.com/{{ commit.github.username }}){%- endif %}\ - {% endfor %} -{% endfor %}\n -""" -# template for the changelog footer -footer = """ - -""" -# remove the leading and trailing s -trim = true -# postprocessors -postprocessors = [ - { pattern = '', replace = "https://github.com/apify/crawlee-python" }, # replace repository URL -] - -[bump] -# With 0.x.y version, breaking commits should only increase the minor version and feature commits should only increase the patch version -breaking_always_bump_major = false -features_always_bump_minor = false - -[git] -# parse the commits based on https://www.conventionalcommits.org -conventional_commits = true -# filter out the commits that are not conventional -filter_unconventional = true -# process each line of a commit as an individual commit -split_commits = false -# regex for preprocessing the commit messages -commit_preprocessors = [ - # Replace PR and issue numbers in commit messages - { pattern = '.*', replace_command = 'python scripts/preprocess_commit_message.py'}, - # Check spelling of the commit with https://github.com/crate-ci/typos - # If the spelling is incorrect, it will be automatically fixed. - #{ pattern = '.*', replace_command = 'typos --write-changes -' }, -] -# regex for parsing and grouping commits -commit_parsers = [ - { message = "^feat", group = "๐Ÿš€ Features" }, - { message = "^fix|^bug", group = "๐Ÿ› Bug Fixes" }, - # { message = "^doc", group = "๐Ÿ“š Documentation" }, - { message = "^doc", skip = true }, - { message = "^perf", group = "โšก Performance" }, - # { message = "^refactor", group = "๐Ÿšœ Refactor" }, - { message = "^refactor", skip = true }, - # { message = "^style", group = "๐ŸŽจ Styling" }, - { message = "^style", skip = true }, - # { message = "^test", group = "๐Ÿงช Testing" }, - { message = "^test", skip = true }, - { message = "^chore\\(release\\): prepare for", skip = true }, - { message = "^chore\\(deps.*\\)", skip = true }, - { message = "^chore\\(pr\\)", skip = true }, - { message = "^chore\\(pull\\)", skip = true }, - # { message = "^chore|^ci", group = "โš™๏ธ Miscellaneous Tasks" }, - { message = "^chore|^ci", skip = true }, - { body = ".*security", group = "๐Ÿ›ก๏ธ Security" }, - { message = "^revert", group = "โ—€๏ธ Revert" }, -] -# protect breaking changes from being skipped due to matching a skipping commit_parser -protect_breaking_commits = true -# filter out the commits that are not matched by commit parsers -filter_commits = false -# regex for matching git tags -tag_pattern = "v[0-9]+\\." -# sort the tags topologically -topo_order = false -# sort the commits inside sections by oldest/newest order -sort_commits = "oldest" -# limit the number of commits included in the changelog. -# limit_commits = 42 - -[remote.github] -owner = "apify" -repo = "crawlee-python" diff --git a/scripts/fetch_pr_issues.sh b/scripts/fetch_pr_issues.sh deleted file mode 100755 index 4e3b82b33..000000000 --- a/scripts/fetch_pr_issues.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -gh api graphql --paginate --slurp \ - -F owner='apify' \ - -F repo='crawlee-python' \ - -f query=' - query ($owner: String!, $repo: String!, $endCursor: String) { - repository(owner: $owner, name: $repo) { - pullRequests(first: 100, after: $endCursor) { - nodes { - number, - closingIssuesReferences(last: 100) { - nodes { number } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - ' | -jq ' - [ - [.[] | .data.repository.pullRequests.nodes ] - | flatten[] - | { - (.number | tostring): - [.closingIssuesReferences.nodes | .[] | .number] - } - ] | add' > pullRequestIssues.json diff --git a/scripts/preprocess_commit_message.py b/scripts/preprocess_commit_message.py deleted file mode 100644 index 30711496f..000000000 --- a/scripts/preprocess_commit_message.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import json -import re -import subprocess -import sys -from pathlib import Path - -pr_issues_file = Path.cwd() / 'pullRequestIssues.json' - - -def load_pr_issues() -> dict[int, list[int]]: - if pr_issues_file.exists(): - return {int(key): value for key, value in json.load(pr_issues_file.open('r')).items()} - - return {} - - -def issue_link(issue_number: int) -> str: - return f'[#{issue_number}](/issues/{issue_number})' - - -def pr_link(pr_number: int) -> str: - return f'[#{pr_number}](/pull/{pr_number})' - - -def replace_issue_or_pull_request_number(match: re.Match) -> str: - item_number = int(match.group(2)) - - pr_to_issues = load_pr_issues() - - if item_number not in pr_to_issues: - subprocess.check_call(str(Path(__file__).parent / 'fetch_pr_issues.sh')) # noqa: S603 - pr_to_issues = load_pr_issues() - - issue_links = [issue_link(issue_number) for issue_number in pr_to_issues.get(item_number, [])] - - if item_number not in pr_to_issues: - return f'({issue_link(item_number)})' - - if not issue_links: - return f'({pr_link(item_number)})' - - return f'({pr_link(item_number)}, closes {", ".join(issue_links)})' - - -if __name__ == '__main__': - print( - re.sub( - r'\((\w+\s)?#([0-9]+)\)', - repl=replace_issue_or_pull_request_number, - string=sys.stdin.read(), - ) - )