From af72439de741ccb3e008bd3fa24b4c496bd75ce0 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 23 Jul 2023 02:41:07 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/scripts/collect_changes.py | 101 ++++++++++++++ .github/template_gitref | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/nightly.yml | 21 +-- .github/workflows/release.yml | 4 +- .github/workflows/scripts/release.py | 182 ++++++++++++-------------- .github/workflows/update-labels.yml | 3 +- .github/workflows/update_ci.yml | 2 +- 8 files changed, 196 insertions(+), 121 deletions(-) create mode 100755 .ci/scripts/collect_changes.py diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py new file mode 100755 index 000000000..04a099dd7 --- /dev/null +++ b/.ci/scripts/collect_changes.py @@ -0,0 +1,101 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_rpm' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import itertools +import os +import re + +import toml +from git import GitCommandError, Repo +from pkg_resources import parse_version + +# Read Towncrier settings +tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] + +CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") +START_STRING = tc_settings.get( + "start_string", + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n", +) +TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") + + +NAME_REGEX = r".*" +VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)" +DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" +TITLE_REGEX = ( + "(" + + re.escape( + TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") + ) + .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_REGEX) + .replace("DATE_REGEX", DATE_REGEX) + + ")" +) + + +def get_changelog(repo, branch): + return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n" + + +def _tokenize_changes(splits): + assert len(splits) % 3 == 0 + for i in range(len(splits) // 3): + title = splits[3 * i] + version = parse_version(splits[3 * i + 1]) + yield [version, title + splits[3 * i + 2]] + + +def split_changelog(changelog): + preamble, rest = changelog.split(START_STRING, maxsplit=1) + split_rest = re.split(TITLE_REGEX, rest) + return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:])) + + +def main(): + repo = Repo(os.getcwd()) + remote = repo.remotes[0] + branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)] + branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) + branches = [ref.name for ref in branches] + + with open(CHANGELOG_FILE, "r") as f: + main_changelog = f.read() + preamble, main_changes = split_changelog(main_changelog) + old_length = len(main_changes) + + for branch in branches: + print(f"Looking at branch {branch}") + try: + changelog = get_changelog(repo, branch) + except GitCommandError: + print("No changelog found on this branch.") + continue + dummy, changes = split_changelog(changelog) + new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True) + # Now remove duplicates (retain the first one) + main_changes = [new_changes[0]] + for left, right in itertools.pairwise(new_changes): + if left[0] != right[0]: + main_changes.append(right) + + new_length = len(main_changes) + if old_length < new_length: + print(f"{new_length - old_length} new versions have been added.") + with open(CHANGELOG_FILE, "w") as fp: + fp.write(preamble) + for change in main_changes: + fp.write(change[1]) + + repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + + +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index c9e5c495d..341daaf8d 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-232-g10acbaa +2021.08.26-237-g4bbee21 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e602fa25b..e366d5e02 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,7 +4,7 @@ # './plugin-template --github pulp_rpm' to update this file. # # For more info visit https://github.com/pulp/plugin_template -name: "CodeQL" +name: "Rpm CodeQL" on: workflow_dispatch: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cab9c4150..97fa428c8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Rpm Nightly CI/CD +name: Rpm Nightly CI on: schedule: # * is a special character in YAML so you have to quote this string @@ -170,16 +170,16 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 1 + fetch-depth: 0 - uses: actions/setup-python@v3 with: - python-version: "3.8" + python-version: "3.11" - name: Install python dependencies run: | echo ::group::PYDEPS - pip install gitpython requests packaging + pip install gitpython toml echo ::endgroup:: - name: Configure Git with pulpbot name and email @@ -187,23 +187,16 @@ jobs: git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Changelog history - run: python .ci/scripts/changelog.py + - name: Collect changes from all branches + run: python .ci/scripts/collect_changes.py - name: Create Pull Request uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.RELEASE_TOKEN }} - committer: pulpbot - author: pulpbot title: 'Update Changelog' - body: '[noissue]' + body: '' branch: 'changelog/update' - base: main - commit-message: | - Update Changelog - - [noissue] delete-branch: true publish: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c92a78125..a1f8e0616 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: Release Pipeline +name: Rpm Release Pipeline on: workflow_dispatch: inputs: @@ -41,7 +41,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install packaging~=21.3 bandersnatch bump2version gitpython towncrier==19.9.0 wheel + pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests echo ::endgroup:: - name: Configure Git with pulpbot name and email diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index a4f5393b7..c3f342632 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -6,53 +6,35 @@ # For more info visit https://github.com/pulp/plugin_template import argparse -import asyncio import re import os -import shutil import textwrap - -from bandersnatch.mirror import BandersnatchMirror -from bandersnatch.master import Master -from bandersnatch.configuration import BandersnatchConfig +import requests from git import Repo - -from packaging.requirements import Requirement from pathlib import Path -async def get_package_from_pypi(package_name, plugin_path): +def get_package_from_pypi(version, plugin_path): """ Download a package from PyPI. - :param name: name of the package to download from PyPI - :return: String path to the package + :param version: version of the package to download from PyPI + :return: True/False if download was successful """ - config = BandersnatchConfig().config - config["mirror"]["master"] = "https://pypi.org" - config["mirror"]["workers"] = "1" - config["mirror"]["directory"] = plugin_path - if not config.has_section("plugins"): - config.add_section("plugins") - config["plugins"]["enabled"] = "blocklist_release\n" - if not config.has_section("allowlist"): - config.add_section("allowlist") - config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n" - config["allowlist"]["packages"] = "\n".join([package_name]) os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - async with Master("https://pypi.org/") as master: - mirror = BandersnatchMirror(homedir=plugin_path, master=master) - name = Requirement(package_name).name - result = await mirror.synchronize([name]) - package_found = False - - for package in result[name]: - current_path = os.path.join(plugin_path, package) - destination_path = os.path.join(plugin_path, "dist", os.path.basename(package)) - shutil.move(current_path, destination_path) - package_found = True - return package_found + r = requests.get(f"https://pypi.org/pypi/pulp-rpm/{version}/json") + if r.status_code == 200: + metadata = r.json() + for url_data in metadata["urls"]: + filename = url_data["filename"] + r2 = requests.get(url_data["url"]) + if r2.status_code != 200: + raise RuntimeError(f"Failed to download released artifact {filename}") + with open(os.path.join(plugin_path, "dist", filename), "wb") as f: + f.write(r2.content) + return True + return False def create_release_commits(repo, release_version, plugin_path): @@ -74,7 +56,7 @@ def create_release_commits(repo, release_version, plugin_path): # Second commit: release version os.system("bump2version release --allow-dirty") - git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/pulp_rpm/*") git.add(f"{plugin_path}/docs/conf.py") git.add(f"{plugin_path}/setup.py") git.add(f"{plugin_path}/requirements.txt") @@ -93,7 +75,7 @@ def create_release_commits(repo, release_version, plugin_path): if not new_dev_version: raise RuntimeError("Could not detect new dev version ... aborting.") - git.add(f"{plugin_path}/{plugin_name}/*") + git.add(f"{plugin_path}/pulp_rpm/*") git.add(f"{plugin_path}/docs/conf.py") git.add(f"{plugin_path}/setup.py") git.add(f"{plugin_path}/requirements.txt") @@ -132,76 +114,76 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): repo.head.reset(index=True, working_tree=True) # Check if Package is available on PyPI - loop = asyncio.get_event_loop() # noqa - # fmt: off - package_found = asyncio.run( - get_package_from_pypi(f"pulp-rpm=={tag.name}", plugin_path) - ) # noqa - # fmt: on - if not package_found: + if not get_package_from_pypi(tag.name, plugin_path): os.system("python3 setup.py sdist bdist_wheel --python-tag py3") -helper = textwrap.dedent( - """\ - Start the release process. +def main(): + helper = textwrap.dedent( + """\ + Start the release process. - Example: - setup.py on plugin before script: - version="2.0.0.dev" + Example: + setup.py on plugin before script: + version="2.0.0.dev" - $ python .ci/scripts/release.py + $ python .ci/scripts/release.py - setup.py on plugin after script: - version="2.0.1.dev" + setup.py on plugin after script: + version="2.0.1.dev" - """ -) -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) - -parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", -) - -args = parser.parse_args() - -release_version_arg = args.release_version - -release_path = os.path.dirname(os.path.abspath(__file__)) -plugin_path = release_path.split("/.github")[0] - -plugin_name = "pulp_rpm" -version = None -with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - version = re.split("\"|'", line)[1] - if not version: - raise RuntimeError("Could not detect existing version ... aborting.") -release_version = version.replace(".dev", "") - -print(f"\n\nRepo path: {plugin_path}") -repo = Repo(plugin_path) - -release_commit = None -if release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break + """ + ) + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, description=helper + ) + + parser.add_argument( + "release_version", + type=str, + help="The version string for the release.", + ) + + args = parser.parse_args() + + release_version_arg = args.release_version + + release_path = os.path.dirname(os.path.abspath(__file__)) + plugin_path = release_path.split("/.github")[0] + + version = None + with open(f"{plugin_path}/setup.py") as fp: + for line in fp.readlines(): + if "version=" in line: + version = re.split("\"|'", line)[1] + if not version: + raise RuntimeError("Could not detect existing version ... aborting.") + release_version = version.replace(".dev", "") + + print(f"\n\nRepo path: {plugin_path}") + repo = Repo(plugin_path) + + release_commit = None + if release_version != release_version_arg: + # Look for a commit with the requested release version + for commit in repo.iter_commits(): + if f"Release {release_version_arg}\n" in commit.message: + release_commit = commit + release_version = release_version_arg + break + if not release_commit: + raise RuntimeError( + f"The release version {release_version_arg} does not match the .dev version at " + "HEAD. A release commit for such version does not exist." + ) + if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at HEAD. " - "A release commit for such version does not exist." - ) - -if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) -else: - release_commit_sha = release_commit.hexsha -create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) + release_commit_sha = create_release_commits(repo, release_version, plugin_path) + else: + release_commit_sha = release_commit.hexsha + create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index 2df110044..d1e584f61 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -7,7 +7,7 @@ --- -name: Update Labels +name: Rpm Update Labels on: push: branches: @@ -37,4 +37,3 @@ jobs: python3 .github/workflows/scripts/update_backport_labels.py env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 9c569dc06..ec5b8d85a 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -7,7 +7,7 @@ --- -name: CI Update +name: Rpm CI Update on: schedule: # * is a special character in YAML so you have to quote this string