Update CI files

[noissue]

.ci/ansible/Containerfile.j2

@@ -2,27 +2,21 @@ FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }
 
 # Add source directories to container
 {% for item in plugins %}
-{% if item.source.startswith("./") or item.ci_requirements | default(false) %}
 ADD ./{{ item.name }} ./{{ item.name }}
-{% endif %}
 {% endfor %}
 
 # Install python packages
+# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
+# Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
 RUN pip3 install
-{%- for item in plugins -%}
-{%- if item.name == "pulp-certguard" -%}
-{{ " " }}python-dateutil rhsm
-{%- endif -%}
-{{ " " }}{{ item.source }}
-{%- if item.name == "pulpcore" -%}
 {%- if s3_test | default(false) -%}
-[s3]
-{%- elif azure_test | default(false) -%}
-[azure]
-{%- elif gcp_test | default(false) -%}
-[google]
+{{ " " }}git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
+{%- for item in plugins -%}
+{{ " " }}{{ item.source }}
+{%- if item.lowerbounds | default(false) -%}
+{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
 {%- endif -%}
 {%- if item.ci_requirements | default(false) -%}
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-259-ge0f38f0
+2021.08.26-298-g64cad93

.github/workflows/build.yml

@@ -6,32 +6,39 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: Build
+name: "Build"
 on:
   workflow_call:
 
+defaults:
+  run:
+    working-directory: "pulp_ostree"
+
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: "actions/checkout@v4"
         with:
           fetch-depth: 1
-      - uses: actions/setup-python@v4
+          path: "pulp_ostree"
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "3.8"
-      - name: Install python dependencies
+      - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install packaging wheel
+          pip install packaging twine wheel
           echo ::endgroup::
-      - name: Build package
-        run: python3 setup.py sdist bdist_wheel --python-tag py3
-      - name: 'Upload Package whl'
-        uses: actions/upload-artifact@v3
+      - name: "Build package"
+        run: |
+          python3 setup.py sdist bdist_wheel --python-tag py3
+          twine check dist/*
+      - name: "Upload Package whl"
+        uses: "actions/upload-artifact@v3"
         with:
-          name: plugin_package
-          path: dist/
-          if-no-files-found: error
+          name: "plugin_package"
+          path: "pulp_ostree/dist/"
+          if-no-files-found: "error"
           retention-days: 5

.github/workflows/changelog.yml

@@ -6,53 +6,53 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: Ostree changelog update
+name: "Ostree changelog update"
 on:
   push:
     branches:
-      - main
+      - "main"
     paths:
-      - CHANGES.rst
-      - CHANGES.md
+      - "CHANGES.rst"
+      - "CHANGES.md"
   workflow_dispatch:
 
 jobs:
 
   update-changelog:
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
     strategy:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: "actions/checkout@v4"
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v4
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "3.8"
 
-      - name: Install python dependencies
+      - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
           pip install -r doc_requirements.txt
           echo ::endgroup::
 
-      - name: Fake api schema
+      - name: "Fake api schema"
         run: |
           mkdir -p docs/_build/html
           echo "{}" > docs/_build/html/api.json
           mkdir -p docs/_static
           echo "{}" > docs/_static/api.json
-      - name:
+      - name: "Build Docs"
         run: |
-          pip install "Jinja2<3.1"
           make diagrams html
-        working-directory: ./docs
+        working-directory: "./docs"
         env:
           PULP_CONTENT_ORIGIN: "http://localhost/"
 
-      - name: Publish changlog to pulpproject.org
-        run: .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/}
+      - name: "Publish changlog to pulpproject.org"
+        run: |
+          .github/workflows/scripts/publish_docs.sh changelog ${GITHUB_REF##*/}
         env:
-          PULP_DOCS_KEY: ${{ secrets.PULP_DOCS_KEY }}
+          PULP_DOCS_KEY: "${{ secrets.PULP_DOCS_KEY }}"

.github/workflows/ci.yml

@@ -6,68 +6,88 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: Ostree CI
+name: "Ostree CI"
 on: {pull_request: {branches: ['*']}}
 
 concurrency:
   group: ${{ github.ref_name }}-${{ github.workflow }}
   cancel-in-progress: true
 
-jobs:
+defaults:
+  run:
+    working-directory: "pulp_ostree"
 
-  ready-to-ship:
-    runs-on: ubuntu-latest
+jobs:
+  check-commits:
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v4
+      - uses: "actions/checkout@v4"
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+          path: "pulp_ostree"
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "3.8"
-      - name: Install requirements
-        run: pip3 install github
-      - name: Check commit message
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install requests pygithub
+          echo ::endgroup::
+      - name: "Check commit message"
         if: github.event_name == 'pull_request'
         env:
-          PY_COLORS: '1'
-          ANSIBLE_FORCE_COLOR: '1'
-          GITHUB_PULL_REQUEST: ${{ github.event.number }}
-          GITHUB_PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
-          GITHUB_BRANCH: ${{ github.head_ref }}
-          GITHUB_REPO_SLUG: ${{ github.repository }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
-        run: sh .github/workflows/scripts/check_commit.sh
-      - name: Verify requirements files
-        run: python .ci/scripts/check_requirements.py
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
+        run: |
+          .github/workflows/scripts/check_commit.sh
+      - name: "Verify requirements files"
+        run: |
+          python .ci/scripts/check_requirements.py
 
   lint:
     uses: "./.github/workflows/lint.yml"
 
   build:
-    needs: lint
+    needs: "lint"
     uses: "./.github/workflows/build.yml"
 
   test:
-    needs: build
+    needs: "build"
     uses: "./.github/workflows/test.yml"
+
   deprecations:
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
     if: github.base_ref == 'main'
-    needs: test
+    needs: "test"
     steps:
-      - name: Fail on deprecations
+      - name: "Create working directory"
         run: |
-          test -z "${{ needs.test.outputs.deprecations-pulp }}"
-          test -z "${{ needs.test.outputs.deprecations-azure }}"
-          test -z "${{ needs.test.outputs.deprecations-s3 }}"
-          test -z "${{ needs.test.outputs.deprecations-lowerbounds }}"
-      - name: Print deprecations
-        if: failure()
+          mkdir -p "pulp_ostree"
+        working-directory: "."
+      - name: "Download Deprecations"
+        uses: actions/download-artifact@v3
+        with:
+          name: "deprecations"
+          path: "pulp_ostree"
+      - name: "Print deprecations"
         run: |
-          echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d
-          echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d
-          echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d
-          echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d
-
+          cat deprecations-*.txt | sort -u
+          ! cat deprecations-*.txt | grep '[^[:space:]]'
 
+  ready-to-ship:
+    # This is a dummy dependent task to have a single entry for the branch protection rules.
+    runs-on: "ubuntu-latest"
+    needs:
+      - "check-commits"
+      - "lint"
+      - "test"
+    if: "always()"
+    steps:
+      - name: "Collect needed jobs results"
+        working-directory: "."
+        run: |
+          echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result'
+          echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0'
+          echo "CI says: Looks good!"

.github/workflows/codeql-analysis.yml

@@ -30,13 +30,13 @@ jobs:
         language: [ 'python' ]
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/create-branch.yml

@@ -21,26 +21,27 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: "actions/checkout@v4"
         with:
           fetch-depth: 0
-          path: pulp_ostree
+          path: "pulp_ostree"
 
-      - uses: actions/setup-python@v4
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "3.8"
 
-      - name: Install python dependencies
+      - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
           pip install bump2version jinja2 pyyaml
           echo ::endgroup::
 
-      - name: Setting secrets
-        working-directory: pulp_ostree
-        run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+      - name: "Setting secrets"
+        working-directory: "pulp_ostree"
+        run: |
+          python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
         env:
-          SECRETS_CONTEXT: ${{ toJson(secrets) }}
+          SECRETS_CONTEXT: "${{ toJson(secrets) }}"
 
       - name: Determine new branch name
         working-directory: pulp_ostree

.github/workflows/kanban.yml

@@ -58,7 +58,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Find issues linked to a PR
     outputs:
-        linked-issues: ${{ steps.linked-issues.outputs.issues }}
+      linked-issues: ${{ steps.linked-issues.outputs.issues }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.github/workflows/lint.yml

@@ -6,24 +6,37 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 ---
-name: Lint
+name: "Lint"
 on:
   workflow_call:
 
+defaults:
+  run:
+    working-directory: "pulp_ostree"
+
 jobs:
   lint:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: "actions/checkout@v4"
         with:
           fetch-depth: 1
-      - uses: actions/setup-python@v4
+          path: "pulp_ostree"
+
+      - uses: "actions/setup-python@v4"
         with:
           python-version: "3.8"
-      # lint_requirements contains tools needed for flake8, etc.
-      - name: Install requirements
-        run: pip3 install -r lint_requirements.txt
+
+      - name: "Install python dependencies"
+        run: |
+          echo ::group::PYDEPS
+          pip install -r lint_requirements.txt
+          echo ::endgroup::
+
+      - name: Lint workflow files
+        run: |
+          yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows
 
       # run black separately from flake8 to get a diff
       - name: Run black