From ec719728dc271579a0082f6d1a9538c33ba4e5fd Mon Sep 17 00:00:00 2001 From: pulpbot Date: Tue, 9 Jul 2024 17:08:15 +0000 Subject: [PATCH] Update CI files [noissue] --- .github/template_gitref | 2 +- .github/workflows/changelog.yml | 58 ----- .github/workflows/nightly.yml | 52 ---- .github/workflows/publish.yml | 47 ---- .github/workflows/scripts/docs-publisher.py | 262 -------------------- .github/workflows/scripts/publish_docs.sh | 48 ---- .github/workflows/scripts/script.sh | 11 - .github/workflows/test.yml | 9 - doc_requirements.txt | 11 - docs/template_gitref | 2 +- template_config.yml | 10 +- 11 files changed, 5 insertions(+), 507 deletions(-) delete mode 100644 .github/workflows/changelog.yml delete mode 100755 .github/workflows/scripts/docs-publisher.py delete mode 100755 .github/workflows/scripts/publish_docs.sh diff --git a/.github/template_gitref b/.github/template_gitref index 19b4c781..b222a637 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-349-gba81617 +2021.08.26-351-g4e8a1a7 diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml deleted file mode 100644 index baeba1d3..00000000 --- a/.github/workflows/changelog.yml +++ /dev/null @@ -1,58 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - ---- -name: "Ostree changelog update" -on: - push: - branches: - - "main" - paths: - - "CHANGES.rst" - - "CHANGES.md" - workflow_dispatch: - -jobs: - - update-changelog: - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install -r doc_requirements.txt - echo ::endgroup:: - - - name: "Fake api schema" - run: | - mkdir -p docs/_build/html - echo "{}" > docs/_build/html/api.json - mkdir -p docs/_static - echo "{}" > docs/_static/api.json - - name: "Build Docs" - run: | - make diagrams html - working-directory: "./docs" - env: - PULP_CONTENT_ORIGIN: "http://localhost/" - - - name: "Publish changlog to pulpproject.org" - run: | - .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/} - env: - PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b7ccf055..ceba8b24 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -65,55 +65,3 @@ jobs: branch: "changelog/update" delete-branch: true path: "pulp_ostree" - - publish: - runs-on: ubuntu-latest - needs: test - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_ostree" - - - uses: actions/download-artifact@v4 - with: - name: "plugin_package" - path: "pulp_ostree/dist/" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: "Set environment variables" - run: | - echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - - - name: Download built docs - uses: actions/download-artifact@v4 - with: - name: "docs.tar" - path: "pulp_ostree" - - - name: Download Python client docs - uses: actions/download-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_ostree" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar -C ./docs - .github/workflows/scripts/publish_docs.sh nightly ${GITHUB_REF##*/} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f59a3fca..e5fb362d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -137,52 +137,6 @@ jobs: - name: "Publish client to rubygems" run: | bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_ostree" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v4" - with: - name: "docs.tar" - path: "pulp_ostree/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v4" - with: - name: "python-client-docs.tar" - path: "pulp_ostree/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} create-gh-release: runs-on: "ubuntu-latest" @@ -191,7 +145,6 @@ jobs: - "publish-package" - "publish-python-bindings" - "publish-ruby-bindings" - - "publish-docs" steps: - name: "Create release on GitHub" diff --git a/.github/workflows/scripts/docs-publisher.py b/.github/workflows/scripts/docs-publisher.py deleted file mode 100755 index 4deed10b..00000000 --- a/.github/workflows/scripts/docs-publisher.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import subprocess -import os -import re -from shutil import rmtree -import tempfile -import requests -import json -from packaging import version - -WORKING_DIR = os.environ["WORKSPACE"] - -VERSION_REGEX = r"(\s*)(version)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" -RELEASE_REGEX = r"(\s*)(release)(\s*)(=)(\s*)(['\"])(.*)(['\"])(.*)" - -USERNAME = "doc_builder_pulp_ostree" -HOSTNAME = "8.43.85.236" - -SITE_ROOT = "/var/www/docs.pulpproject.org/pulp_ostree/" - - -def make_directory_with_rsync(remote_paths_list): - """ - Ensure the remote directory path exists. - - :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the - remote. - :type remote_paths_list: a list of strings, with each string representing a directory. - """ - try: - tempdir_path = tempfile.mkdtemp() - cwd = os.getcwd() - os.chdir(tempdir_path) - os.makedirs(os.sep.join(remote_paths_list)) - remote_path_arg = "%s@%s:%s%s" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - remote_paths_list[0], - ) - local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep - rsync_command = ["rsync", "-avzh", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command) - if exit_code != 0: - raise RuntimeError("An error occurred while creating remote directories.") - finally: - rmtree(tempdir_path) - os.chdir(cwd) - - -def ensure_dir(target_dir, clean=True): - """ - Ensure that the directory specified exists and is empty. - - By default this will delete the directory if it already exists - - :param target_dir: The directory to process - :type target_dir: str - :param clean: Whether or not the directory should be removed and recreated - :type clean: bool - """ - if clean: - rmtree(target_dir, ignore_errors=True) - try: - os.makedirs(target_dir) - except OSError: - pass - - -def main(): - """ - Builds documentation using the 'make html' command and rsyncs to docs.pulpproject.org. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--build-type", required=True, help="Build type: nightly, tag or changelog." - ) - parser.add_argument("--branch", required=True, help="Branch or tag name.") - opts = parser.parse_args() - - build_type = opts.build_type - - branch = opts.branch - - publish_at_root = False - - # rsync the docs - print("rsync the docs") - docs_directory = os.sep.join([WORKING_DIR, "docs"]) - local_path_arg = os.sep.join([docs_directory, "_build", "html"]) + os.sep - if build_type == "nightly": - # This is a nightly build - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - make_directory_with_rsync(["en", branch, build_type]) - rsync_command = ["rsync", "-avzh", "--delete", local_path_arg, remote_path_arg] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "tag": - if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: - # Only publish docs at the root if this is the latest version - r = requests.get("https://pypi.org/pypi/pulp-ostree/json") - latest_version = version.parse(json.loads(r.text)["info"]["version"]) - docs_version = version.parse(branch) - # This is to mitigate delays on PyPI which doesn't update metadata in timely manner. - # It doesn't prevent incorrect docs being published at root if 2 releases are done close - # to each other, within PyPI delay. E.g. Release 3.11.0 an then 3.10.1 immediately - # after. - if docs_version >= latest_version: - publish_at_root = True - # Post releases should use the x.y.z part of the version string to form a path - if "post" in branch: - branch = ".".join(branch.split(".")[:-1]) - - # This is a GA build. - # publish to the root of docs.pulpproject.org - if publish_at_root: - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - # This is a pre-release - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - branch, - build_type, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "nightly", - "--exclude", - "testing", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y/ - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - make_directory_with_rsync(["en", x_y_version]) - remote_path_arg = "%s@%s:%sen/%s/" % ( - USERNAME, - HOSTNAME, - SITE_ROOT, - x_y_version, - ) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - # publish to docs.pulpproject.org/en/3.y.z/ - make_directory_with_rsync(["en", branch]) - remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - elif build_type == "changelog": - if branch != "main": - raise RuntimeError("Can only publish CHANGELOG from main") - # Publish the CHANGELOG from main branch at the root directory - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - changelog_local_path_arg = os.path.join(local_path_arg, "changes.html") - rsync_command = [ - "rsync", - "-vzh", - "--omit-dir-times", - changelog_local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") - else: - raise RuntimeError("Build type must be either 'nightly', 'tag' or 'changelog'.") - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/publish_docs.sh b/.github/workflows/scripts/publish_docs.sh deleted file mode 100755 index f92e1873..00000000 --- a/.github/workflows/scripts/publish_docs.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -mkdir ~/.ssh -touch ~/.ssh/pulp-infra -chmod 600 ~/.ssh/pulp-infra -echo "$PULP_DOCS_KEY" > ~/.ssh/pulp-infra - -echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> ~/.ssh/known_hosts -chmod 644 ~/.ssh/known_hosts - -export PYTHONUNBUFFERED=1 -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py -export WORKSPACE=$PWD - -# start the ssh agent -eval "$(ssh-agent -s)" -ssh-add ~/.ssh/pulp-infra - -python3 .github/workflows/scripts/docs-publisher.py --build-type "$1" --branch "$2" - -if [[ "$GITHUB_WORKFLOW" == "Ostree changelog update" ]]; then - # Do not build bindings docs on changelog update - exit -fi - -mkdir -p ../ostree-bindings -tar -xvf ostree-python-client-docs.tar --directory ../ostree-bindings -pushd ../ostree-bindings - -# publish to docs.pulpproject.org/pulp_ostree_client -rsync -avzh site/ doc_builder_pulp_ostree@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_ostree_client/ - -# publish to docs.pulpproject.org/pulp_ostree_client/en/{release} -rsync -avzh site/ doc_builder_pulp_ostree@docs.pulpproject.org:/var/www/docs.pulpproject.org/pulp_ostree_client/en/"$2" -popd diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 8e670d69..5bfef882 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -16,7 +16,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh -export POST_DOCS_TEST=$PWD/.github/workflows/scripts/post_docs_test.sh export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh # Needed for both starting the service and building the docs. @@ -31,17 +30,7 @@ if [[ "$TEST" = "docs" ]]; then if [[ "$GITHUB_WORKFLOW" == "Ostree CI" ]]; then towncrier build --yes --version 4.0.0.ci fi - # Unified Docs Build pulp-docs build - # Legacy Docs Build - cd docs - make PULP_URL="$PULP_URL" diagrams html - tar -cvf docs.tar ./_build - cd .. - - if [ -f "$POST_DOCS_TEST" ]; then - source "$POST_DOCS_TEST" - fi exit fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b334b6c9..2248f9d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -137,15 +137,6 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true - - name: Upload built docs - if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v4 - with: - name: "docs.tar" - path: "pulp_ostree/docs/docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - name: "Logs" if: always() diff --git a/doc_requirements.txt b/doc_requirements.txt index 1768d1fc..84f0533c 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -7,15 +7,4 @@ -r requirements.txt towncrier -# Legacy docs -plantuml -sphinx~=7.1.2 -sphinx-rtd-theme==1.3.0 -sphinxcontrib-jquery -sphinxcontrib-openapi -mistune<4.0.0 -Jinja2<3.2 - -# Unified docs pulp-docs @ git+https://github.com/pulp/pulp-docs@main -# Extra requirements diff --git a/docs/template_gitref b/docs/template_gitref index 19b4c781..b222a637 100644 --- a/docs/template_gitref +++ b/docs/template_gitref @@ -1 +1 @@ -2021.08.26-349-gba81617 +2021.08.26-351-g4e8a1a7 diff --git a/template_config.yml b/template_config.yml index 7a8944bc..d9feaaf1 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-347-gc4a2504 +# generated with plugin_template@2021.08.26-351-g4e8a1a7 api_root: /pulp/ black: true @@ -23,7 +23,6 @@ disabled_redis_runners: [] doc_requirements_from_pulpcore: false docker_fixtures: false docs_test: true -extra_docs_requirements: [] flake8: true flake8_ignore: [] github_org: pulp @@ -33,8 +32,8 @@ latest_release_branch: '2.3' lint_requirements: true noissue_marker: '[noissue]' os_required_packages: - - libgirepository1.0-dev - - libostree-dev +- libgirepository1.0-dev +- libostree-dev parallel_test_workers: 8 plugin_app_label: ostree plugin_default_branch: main @@ -44,7 +43,6 @@ plugins: name: pulp_ostree post_job_template: null pre_job_template: null -publish_docs_to_pulpprojectdotorg: true pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} @@ -73,6 +71,4 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true -use_legacy_docs: true -use_unified_docs: true