From bf17f0d11bd2ea31adcc91f2cf0c82e865b788c1 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 7 Jul 2024 02:40:14 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 1 + .ci/assets/ci_constraints.txt | 7 ++ .github/template_gitref | 2 +- .github/workflows/build.yml | 65 +++++++++- .github/workflows/publish.yml | 112 +----------------- .../workflows/scripts/build_python_client.sh | 19 +++ .../workflows/scripts/build_ruby_client.sh | 19 +++ .../scripts/install_python_client.sh | 27 ----- .../workflows/scripts/install_ruby_client.sh | 27 ----- .github/workflows/scripts/script.sh | 109 ++++++++--------- .github/workflows/test.yml | 32 +++-- template_config.yml | 3 +- 12 files changed, 189 insertions(+), 234 deletions(-) create mode 100644 .ci/assets/ci_constraints.txt create mode 100755 .github/workflows/scripts/build_python_client.sh create mode 100755 .github/workflows/scripts/build_ruby_client.sh delete mode 100755 .github/workflows/scripts/install_python_client.sh delete mode 100755 .github/workflows/scripts/install_ruby_client.sh diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 338ff697..bad75f3b 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -22,6 +22,7 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt new file mode 100644 index 00000000..2617a408 --- /dev/null +++ b/.ci/assets/ci_constraints.txt @@ -0,0 +1,7 @@ +# Pulpcore versions without the openapi command do no longer work in the CI +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* + + + +tablib!=3.6.0 +# 3.6.0: This release introduced a regression removing the "html" optional dependency. diff --git a/.github/template_gitref b/.github/template_gitref index 1d4100e7..19b4c781 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-341-g5cb6e90 +2021.08.26-349-gba81617 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 968b41df..2d355a20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,18 +23,29 @@ jobs: with: fetch-depth: 1 path: "pulp-certguard" + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | python3 setup.py sdist bdist_wheel --python-tag py3 twine check dist/* + - name: "Install built packages" + run: | + pip install dist/pulp_certguard-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt + - name: "Generate api specs" + run: | + pulpcore-manager openapi --file "api.json" - name: "Upload Package whl" uses: "actions/upload-artifact@v4" with: @@ -43,3 +54,55 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + - name: "Upload API specs" + uses: "actions/upload-artifact@v4" + with: + name: "api_spec" + path: | + pulp-certguard/api.json + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Python bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Build Ruby bindings packages" + run: | + .github/workflows/scripts/build_ruby_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 31f68240..d218dab9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,114 +19,10 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" - - build-bindings-docs: - needs: - - "build" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp-certguard" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v5" - with: - python-version: "3.11" - - - uses: "actions/download-artifact@v4" - with: - name: "plugin_package" - path: "pulp-certguard/dist/" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp-certguard/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" - with: - name: "python-client.tar" - path: | - if-no-files-found: "error" - overwrite: true - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" - with: - name: "python-client-docs.tar" - path: | - if-no-files-found: "error" - overwrite: true - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v4" - with: - name: "ruby-client.tar" - path: | - if-no-files-found: "error" - overwrite: true - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -164,7 +60,7 @@ jobs: publish-python-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -206,7 +102,7 @@ jobs: publish-ruby-bindings: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -243,7 +139,7 @@ jobs: create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" - "publish-package" - "publish-python-bindings" - "publish-ruby-bindings" diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh new file mode 100755 index 00000000..1543840e --- /dev/null +++ b/.github/workflows/scripts/build_python_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_certguard' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +popd diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh new file mode 100755 index 00000000..fcf08f76 --- /dev/null +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -ruby-client.tar file in the plugins root directory. + +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_certguard' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +popd diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh deleted file mode 100755 index 1cf1f8bf..00000000 --- a/.github/workflows/scripts/install_python_client.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "certguard" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -popd diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh deleted file mode 100755 index 1cf1f8bf..00000000 --- a/.github/workflows/scripts/install_ruby_client.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_certguard' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -set -mveuo pipefail - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -source .github/workflows/scripts/utils.sh - -PULP_URL="${PULP_URL:-https://pulp}" -export PULP_URL -PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" -export PULP_API_ROOT - -REPORTED_STATUS="$(pulp status)" -REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "certguard" -r '.versions[] | select(.component == $plugin) | .version')" -VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')" - -pushd ../pulp-openapi-generator -popd diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 5db22ac6..99f92dcd 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -43,75 +43,70 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Generate and install binding +# Generate bindings +################### + +echo "::group::Generate bindings" + +touch bindings_requirements.txt pushd ../pulp-openapi-generator -if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" -then # Use app_label to generate api.json and package to produce the proper package name. - if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + # Workaround: Domains are not supported by the published bindings. + # Sadly: Different pulpcore-versions aren't either... + # So we exclude the prebuilt ones only for domains disabled. + if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS="" else - # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[]| tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done + BUILT_CLIENTS=" " fi -else - # Infer the client name from the package name by replacing "-" with "_". - # Use the component to infer the package name on older versions of pulpcore. - if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - fi -fi + for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") + do + COMPONENT="$(jq -r '.component' <<<"${ITEM}")" + VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")" + # On older status endpoints, the module was not provided, but the package should be accurate + # there, because we did not merge plugins into pulpcore back then. + MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")" + PACKAGE="${MODULE%%.*}" + cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json" + if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]] + then + rm -rf "./${PACKAGE}-client" + ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}" + pushd "${PACKAGE}-client" + python setup.py sdist bdist_wheel --python-tag py3 + popd + else + if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ] + then + ls -lR "${PACKAGE}-client/" + echo "Error: Client bindings for ${COMPONENT} not found." + echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing." + exit 1 + fi + fi + echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp-certguard/bindings_requirements.txt" + done popd -# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. -echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt +echo "::endgroup::" + +echo "::group::Debug bindings diffs" + +echo "::endgroup::" + +# Install test requirements +########################### + +# Add a safeguard to make sure the proper versions of the clients are installed. +echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt -cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt +cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt +cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())') cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf13a3b4..b4ddc336 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,15 +39,32 @@ jobs: with: python-version: "3.11" - - uses: "actions/download-artifact@v4" + - name: "Download plugin package" + uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp-certguard/dist/" + - name: "Download API specs" + uses: "actions/download-artifact@v4" + with: + name: "api_spec" + path: "pulp-certguard/" + + - name: "Download client packages" + uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp-certguard" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs jq jsonpatch echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp-certguard/.ci/assets/httpie/" >> $GITHUB_ENV echo ::endgroup:: @@ -90,16 +107,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('s3', matrix.env.TEST) }}" - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - if: "${{ env.TEST == 'pulp' }}" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Script" run: | .github/workflows/scripts/script.sh @@ -121,3 +128,4 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +... diff --git a/template_config.yml b/template_config.yml index e56f8913..3aeebef7 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-341-g5cb6e90 +# generated with plugin_template@2021.08.26-349-gba81617 api_root: /pulp/ black: true @@ -33,6 +33,7 @@ kanban: true latest_release_branch: null lint_requirements: false noissue_marker: '[noissue]' +os_required_packages: [] parallel_test_workers: 8 plugin_app_label: certguard plugin_default_branch: main