diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 338ff697..afe0b118 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,12 +9,17 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} {%- if item.lowerbounds | default(false) -%} {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} @@ -22,6 +27,8 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 00000000..2617a408 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,7 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 00000000..ab8a9b8b --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,118 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_certguard' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro-1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor-1}" + elif version.major != 0: + max_version = f"{version.major-1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + else: + operator = "==" + min_version = spec.version + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="store_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + with fileinput.input(files=args.filename) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + if args.upper: + new_req = to_upper_bound(req) + else: + new_req = to_lower_bound(req) + print(new_req + comment) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 33c1d88f..00000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.github/template_gitref b/.github/template_gitref index 1d4100e7..e1ea9541 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-341-g5cb6e90 +2021.08.26-362-g5add0ff diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 968b41df..a79ca4b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,18 +23,29 @@ jobs: with: fetch-depth: 1 path: "pulp-certguard" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | python3 setup.py sdist bdist_wheel --python-tag py3 twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_certguard-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" - name: "Upload Package whl" uses: "actions/upload-artifact@v4" with: @@ -43,3 +54,13 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v4" + with: + name: "api_spec" + path: | + pulp-certguard/api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index 35f99782..00000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: "Certguard changelog update" -on: - push: - branches: - - "main" - paths: - - "CHANGES.rst" - - "CHANGES.md" - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: "Fake api schema" - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: "Build Docs" - run: | - make diagrams html - working-directory: "./docs" - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: "Publish changlog to pulpproject.org" - run: | - .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1f315663..9906dd1d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,32 +65,3 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp-certguard" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp-certguard" - - - uses: actions/download-artifact@v4 - with: - name: "plugin_package" - path: "pulp-certguard/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 31f68240..3a0eb3b9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,114 +19,10 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" - - build-bindings-docs: - needs: - - "build" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp-certguard" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - uses: "actions/download-artifact@v4" - with: - name: "plugin_package" - path: "pulp-certguard/dist/" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp-certguard/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" - with: - name: "python-client.tar" - path: | - if-no-files-found: "error" - overwrite: true - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" - with: - name: "python-client-docs.tar" - path: | - if-no-files-found: "error" - overwrite: true - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v4" - with: - name: "ruby-client.tar" - path: | - if-no-files-found: "error" - overwrite: true - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -161,92 +57,12 @@ jobs: - name: "Deploy plugin to pypi" run: | .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} - publish-python-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp-certguard" - - - name: "Download Python client" - uses: "actions/download-artifact@v4" - with: - name: "python-client.tar" - path: "pulp-certguard/" - - - name: "Untar python client packages" - run: | - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} - publish-ruby-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp-certguard" - - - name: "Download Ruby client" - uses: "actions/download-artifact@v4" - with: - name: "ruby-client.tar" - path: "pulp-certguard/" - - - name: "Untar Ruby client packages" - run: | - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to rubygems" - run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" - "publish-package" - - "publish-python-bindings" - - "publish-ruby-bindings" steps: - name: "Create release on GitHub" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 95ecf600..a2974606 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -65,9 +65,11 @@ then exit $s fi +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh new file mode 100755 index 00000000..1543840e --- /dev/null +++ b/.github/workflows/scripts/build_python_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_certguard' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +popd diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 00000000..fcf08f76 --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_certguard' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +popd diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index f851717a..00000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_certguard" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_certguard/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-certguard/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index a949ae76..984024c3 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -45,6 +45,11 @@ if [[ -f ../../ci_requirements.txt ]]; then ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi cat >> vars/main.yaml << VARSYAML services: diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh deleted file mode 100755 index 1cf1f8bf..00000000 --- a/.github/workflows/scripts/install_python_client.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "certguard" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 1cf1f8bf..00000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "certguard" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -popd diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index a28a7194..00000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Certguard changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 5db22ac6..f83e88e7 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -16,7 +16,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh # Needed for both starting the service and building the docs. @@ -31,6 +30,7 @@ if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Certguard CI" ]]; then towncrier build --yes --version 4.0.0.ci fi + pulp-docs build exit fi @@ -43,75 +43,70 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Generate and install binding +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt pushd ../pulp-openapi-generator -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then # Use app_label to generate api.json and package to produce the proper package name. - if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # So we exclude the prebuilt ones only for domains disabled. + if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS="" else - # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[]| tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS=" " fi -else - # Infer the client name from the package name by replacing "-" with "_". - # Use the component to infer the package name on older versions of pulpcore. - if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - fi -fi + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") + do + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] + then + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd + else + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi + fi + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp-certguard/bindings_requirements.txt" + done popd -# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +echo "::endgroup::" + +# Install test requirements +########################### + +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf13a3b4..d6c29d03 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,15 +39,22 @@ jobs: with: python-version: "3.11" - - uses: "actions/download-artifact@v4" + - name: "Download plugin package" + uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp-certguard/dist/" + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp-certguard/" + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp-certguard/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -90,16 +97,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('s3', matrix.env.TEST) }}" - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - if: "${{ env.TEST == 'pulp' }}" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Script" run: | .github/workflows/scripts/script.sh @@ -121,3 +118,4 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +... diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 40ada1da..75134175 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -79,7 +79,7 @@ jobs: - name: "Run update" working-directory: "pulp-certguard" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -106,7 +106,7 @@ jobs: - name: "Run update" working-directory: "pulp-certguard" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" @@ -133,7 +133,7 @@ jobs: - name: "Run update" working-directory: "pulp-certguard" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" diff --git a/template_config.yml b/template_config.yml index e56f8913..81bb96aa 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-341-g5cb6e90 +# generated with plugin_template@2021.08.26-362-g5add0ff api_root: /pulp/ black: true @@ -24,7 +24,6 @@ disabled_redis_runners: doc_requirements_from_pulpcore: false docker_fixtures: false docs_test: false -extra_docs_requirements: [] flake8: true flake8_ignore: [] github_org: pulp @@ -33,6 +32,7 @@ kanban: true latest_release_branch: null lint_requirements: false noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: certguard plugin_default_branch: main @@ -40,7 +40,6 @@ plugin_name: pulp-certguard plugins: [] post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: false pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} @@ -71,6 +70,4 @@ test_performance: false test_reroute: false test_s3: false use_issue_template: true -use_legacy_docs: false -use_unified_docs: false