Update CI files for branch main

.ci/scripts/collect_changes.py

@@ -0,0 +1,101 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_certguard' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import itertools
+import os
+import re
+
+import toml
+from git import GitCommandError, Repo
+from pkg_resources import parse_version
+
+# Read Towncrier settings
+tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+
+CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
+START_STRING = tc_settings.get(
+    "start_string",
+    "<!-- towncrier release notes start -->\n"
+    if CHANGELOG_FILE.endswith(".md")
+    else ".. towncrier release notes start\n",
+)
+TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
+
+
+NAME_REGEX = r".*"
+VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
+TITLE_REGEX = (
+    "("
+    + re.escape(
+        TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
+    )
+    .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_REGEX)
+    .replace("DATE_REGEX", DATE_REGEX)
+    + ")"
+)
+
+
+def get_changelog(repo, branch):
+    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+
+
+def _tokenize_changes(splits):
+    assert len(splits) % 3 == 0
+    for i in range(len(splits) // 3):
+        title = splits[3 * i]
+        version = parse_version(splits[3 * i + 1])
+        yield [version, title + splits[3 * i + 2]]
+
+
+def split_changelog(changelog):
+    preamble, rest = changelog.split(START_STRING, maxsplit=1)
+    split_rest = re.split(TITLE_REGEX, rest)
+    return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
+
+
+def main():
+    repo = Repo(os.getcwd())
+    remote = repo.remotes[0]
+    branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)]
+    branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
+    branches = [ref.name for ref in branches]
+
+    with open(CHANGELOG_FILE, "r") as f:
+        main_changelog = f.read()
+    preamble, main_changes = split_changelog(main_changelog)
+    old_length = len(main_changes)
+
+    for branch in branches:
+        print(f"Looking at branch {branch}")
+        try:
+            changelog = get_changelog(repo, branch)
+        except GitCommandError:
+            print("No changelog found on this branch.")
+            continue
+        dummy, changes = split_changelog(changelog)
+        new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
+        # Now remove duplicates (retain the first one)
+        main_changes = [new_changes[0]]
+        for left, right in itertools.pairwise(new_changes):
+            if left[0] != right[0]:
+                main_changes.append(right)
+
+    new_length = len(main_changes)
+    if old_length < new_length:
+        print(f"{new_length - old_length} new versions have been added.")
+        with open(CHANGELOG_FILE, "w") as fp:
+            fp.write(preamble)
+            for change in main_changes:
+                fp.write(change[1])
+
+        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+
+
+if __name__ == "__main__":
+    main()

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-233-gd6a6b5f
+2021.08.26-237-g4bbee21

.github/workflows/nightly.yml

@@ -170,40 +170,33 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 1
+          fetch-depth: 0
 
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: Install python dependencies
         run: |
           echo ::group::PYDEPS
-          pip install gitpython requests packaging
+          pip install gitpython toml
           echo ::endgroup::
 
       - name: Configure Git with pulpbot name and email
         run: |
           git config --global user.name 'pulpbot'
           git config --global user.email 'pulp-infra@redhat.com'
 
-      - name: Changelog history
-        run: python .ci/scripts/changelog.py
+      - name: Collect changes from all branches
+        run: python .ci/scripts/collect_changes.py
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v4
         with:
           token: ${{ secrets.RELEASE_TOKEN }}
-          committer: pulpbot <pulp-infra@redhat.com>
-          author: pulpbot <pulp-infra@redhat.com>
           title: 'Update Changelog'
-          body: '[noissue]'
+          body: ''
           branch: 'changelog/update'
-          base: main
-          commit-message: |
-            Update Changelog
-
-            [noissue]
           delete-branch: true
 
   publish:

.github/workflows/release.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Install python dependencies
         run: |
           echo ::group::PYDEPS
-          pip install packaging~=21.3 bandersnatch bump2version gitpython towncrier==19.9.0 wheel
+          pip install packaging~=21.3 bump2version gitpython towncrier==19.9.0 wheel requests
           echo ::endgroup::
 
       - name: Configure Git with pulpbot name and email

.github/workflows/scripts/release.py

@@ -6,53 +6,35 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import argparse
-import asyncio
 import re
 import os
-import shutil
 import textwrap
-
-from bandersnatch.mirror import BandersnatchMirror
-from bandersnatch.master import Master
-from bandersnatch.configuration import BandersnatchConfig
+import requests
 
 from git import Repo
-
-from packaging.requirements import Requirement
 from pathlib import Path
 
 
-async def get_package_from_pypi(package_name, plugin_path):
+def get_package_from_pypi(version, plugin_path):
     """
     Download a package from PyPI.
 
-    :param name: name of the package to download from PyPI
-    :return: String path to the package
+    :param version: version of the package to download from PyPI
+    :return: True/False if download was successful
     """
-    config = BandersnatchConfig().config
-    config["mirror"]["master"] = "https://pypi.org"
-    config["mirror"]["workers"] = "1"
-    config["mirror"]["directory"] = plugin_path
-    if not config.has_section("plugins"):
-        config.add_section("plugins")
-    config["plugins"]["enabled"] = "blocklist_release\n"
-    if not config.has_section("allowlist"):
-        config.add_section("allowlist")
-    config["plugins"]["enabled"] += "allowlist_release\nallowlist_project\n"
-    config["allowlist"]["packages"] = "\n".join([package_name])
     os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True)
-    async with Master("https://pypi.org/") as master:
-        mirror = BandersnatchMirror(homedir=plugin_path, master=master)
-        name = Requirement(package_name).name
-        result = await mirror.synchronize([name])
-    package_found = False
-
-    for package in result[name]:
-        current_path = os.path.join(plugin_path, package)
-        destination_path = os.path.join(plugin_path, "dist", os.path.basename(package))
-        shutil.move(current_path, destination_path)
-        package_found = True
-    return package_found
+    r = requests.get(f"https://pypi.org/pypi/pulp-certguard/{version}/json")
+    if r.status_code == 200:
+        metadata = r.json()
+        for url_data in metadata["urls"]:
+            filename = url_data["filename"]
+            r2 = requests.get(url_data["url"])
+            if r2.status_code != 200:
+                raise RuntimeError(f"Failed to download released artifact {filename}")
+            with open(os.path.join(plugin_path, "dist", filename), "wb") as f:
+                f.write(r2.content)
+        return True
+    return False
 
 
 def create_release_commits(repo, release_version, plugin_path):
@@ -74,7 +56,7 @@ def create_release_commits(repo, release_version, plugin_path):
     # Second commit: release version
     os.system("bump2version release --allow-dirty")
 
-    git.add(f"{plugin_path}/{plugin_name}/*")
+    git.add(f"{plugin_path}/pulp_certguard/*")
     git.add(f"{plugin_path}/docs/conf.py")
     git.add(f"{plugin_path}/setup.py")
     git.add(f"{plugin_path}/requirements.txt")
@@ -93,7 +75,7 @@ def create_release_commits(repo, release_version, plugin_path):
         if not new_dev_version:
             raise RuntimeError("Could not detect new dev version ... aborting.")
 
-    git.add(f"{plugin_path}/{plugin_name}/*")
+    git.add(f"{plugin_path}/pulp_certguard/*")
     git.add(f"{plugin_path}/docs/conf.py")
     git.add(f"{plugin_path}/setup.py")
     git.add(f"{plugin_path}/requirements.txt")
@@ -132,76 +114,76 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path):
     repo.head.reset(index=True, working_tree=True)
 
     # Check if Package is available on PyPI
-    loop = asyncio.get_event_loop()  # noqa
-    # fmt: off
-    package_found = asyncio.run(
-        get_package_from_pypi(f"pulp-certguard=={tag.name}", plugin_path)
-    )  # noqa
-    # fmt: on
-    if not package_found:
+    if not get_package_from_pypi(tag.name, plugin_path):
         os.system("python3 setup.py sdist bdist_wheel --python-tag py3")
 
 
-helper = textwrap.dedent(
-    """\
-        Start the release process.
+def main():
+    helper = textwrap.dedent(
+        """\
+            Start the release process.
 
-        Example:
-            setup.py on plugin before script:
-                version="2.0.0.dev"
+            Example:
+                setup.py on plugin before script:
+                    version="2.0.0.dev"
 
-            $ python .ci/scripts/release.py
+                $ python .ci/scripts/release.py
 
-            setup.py on plugin after script:
-                version="2.0.1.dev"
+                setup.py on plugin after script:
+                    version="2.0.1.dev"
 
 
-    """
-)
-parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper)
-
-parser.add_argument(
-    "release_version",
-    type=str,
-    help="The version string for the release.",
-)
-
-args = parser.parse_args()
-
-release_version_arg = args.release_version
-
-release_path = os.path.dirname(os.path.abspath(__file__))
-plugin_path = release_path.split("/.github")[0]
-
-plugin_name = "pulp_certguard"
-version = None
-with open(f"{plugin_path}/setup.py") as fp:
-    for line in fp.readlines():
-        if "version=" in line:
-            version = re.split("\"|'", line)[1]
-    if not version:
-        raise RuntimeError("Could not detect existing version ... aborting.")
-release_version = version.replace(".dev", "")
-
-print(f"\n\nRepo path: {plugin_path}")
-repo = Repo(plugin_path)
-
-release_commit = None
-if release_version != release_version_arg:
-    # Look for a commit with the requested release version
-    for commit in repo.iter_commits():
-        if f"Release {release_version_arg}\n" in commit.message:
-            release_commit = commit
-            release_version = release_version_arg
-            break
+        """
+    )
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawTextHelpFormatter, description=helper
+    )
+
+    parser.add_argument(
+        "release_version",
+        type=str,
+        help="The version string for the release.",
+    )
+
+    args = parser.parse_args()
+
+    release_version_arg = args.release_version
+
+    release_path = os.path.dirname(os.path.abspath(__file__))
+    plugin_path = release_path.split("/.github")[0]
+
+    version = None
+    with open(f"{plugin_path}/setup.py") as fp:
+        for line in fp.readlines():
+            if "version=" in line:
+                version = re.split("\"|'", line)[1]
+        if not version:
+            raise RuntimeError("Could not detect existing version ... aborting.")
+    release_version = version.replace(".dev", "")
+
+    print(f"\n\nRepo path: {plugin_path}")
+    repo = Repo(plugin_path)
+
+    release_commit = None
+    if release_version != release_version_arg:
+        # Look for a commit with the requested release version
+        for commit in repo.iter_commits():
+            if f"Release {release_version_arg}\n" in commit.message:
+                release_commit = commit
+                release_version = release_version_arg
+                break
+        if not release_commit:
+            raise RuntimeError(
+                f"The release version {release_version_arg} does not match the .dev version at "
+                "HEAD. A release commit for such version does not exist."
+            )
+
     if not release_commit:
-        raise RuntimeError(
-            f"The release version {release_version_arg} does not match the .dev version at HEAD. "
-            "A release commit for such version does not exist."
-        )
-
-if not release_commit:
-    release_commit_sha = create_release_commits(repo, release_version, plugin_path)
-else:
-    release_commit_sha = release_commit.hexsha
-create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path)
+        release_commit_sha = create_release_commits(repo, release_version, plugin_path)
+    else:
+        release_commit_sha = release_commit.hexsha
+    create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path)
+
+
+if __name__ == "__main__":
+    main()