[dev-v2.8] Update scripts/pull-scripts to use new release scheme

.github/pull_request_template.md

@@ -24,3 +24,6 @@
 
 ## Backporting considerations
 <!-- Does this change need to be backported to other versions? If so, which versions should it be backported to? -->
+
+## Validation steps
+- [] Ensure any new images have `repository` and `tag` on the same level to ensure the rancher-images.txt is generated correctly for it for airgap scenarios.

.github/workflows/pull-request.yaml

@@ -8,22 +8,74 @@ on:
 
 jobs:
   build:
+    name: Build
     runs-on: ubuntu-latest
-
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+
+      - name: Checkout PR
+        run: gh pr checkout ${{ github.event.pull_request.number }}
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout into branch
         run: git checkout -b staging-pr-workflow
 
       - name: Pull scripts
         run: sudo make pull-scripts
 
-      - name: Pull in all relevant branches
-        run: git fetch origin release-v2.6
-      
-      - name: Validate 
+      - name: Check release.yaml
+        run: sudo make check-release-yaml
+
+      - name: Validate
         run: sudo make validate
 
+  check-images:
+    name: Check Container Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+
+      - name: Checkout PR
+        run: gh pr checkout ${{ github.event.pull_request.number }}
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check container images
+        run: make check-images
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  check-rc:
+    name: Check RC Images and Charts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+
+      - name: Checkout PR
+        run: gh pr checkout ${{ github.event.pull_request.number }}
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check RC images and charts
+        run: make check-rc
+        if: startsWith(github.ref, 'refs/heads/release-v')
+
+  hull-tests:
+    name: Run Hull tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@v4
+
+      - name: Checkout PR
+        run: gh pr checkout ${{ github.event.pull_request.number }}
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Run Hull tests
-        run: cd tests && go test -v ./...
+        run: cd tests && go test -v ./...

.github/workflows/regsync-config.yaml

@@ -1,38 +1,86 @@
-# Generate-Regsync-Config action will checkout release-v2.7 branch, run make regsync target and
-# creates a pull request from rancherbot/charts or rancher/charts release-v2.7 branch with any image additions
-# to regsync config file. This action is triggered whenever something is pushed into release-v2.7 branch.
+# Generate-Regsync-Config action will run for every PR into release-v2.7 branch only after an approval is given
+# It will run make target to generate regsync file and add a commit to the PR updating the regsync file.
+# It will then install and run regsync client and do the prime image mirroring.
 
 name: Generate-Regsync-Config
 
 on:
-  pull_request_review:
-    types: [submitted, edited]
+  pull_request:
+    types:
+      - labeled
 
 jobs:
+  onLabelAndApproval:
+    if: github.event.label.name == 'regsync-ready' && startsWith(github.event.pull_request.base.ref, 'release-v')
+    runs-on: ubuntu-latest
+    outputs:
+      is_approved: ${{ steps.check-approval.outputs.approved }}
+    steps:
+      - name: Check if PR is approved
+        id: check-approval
+        run: |
+          IS_APPROVED=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews --jq '[.[] | select(.state == "APPROVED")] | length')
+          if [[ "$IS_APPROVED" -gt 0 ]]; then
+            echo "::set-output name=approved::true"
+          else
+            echo "::set-output name=approved::false"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   build:
-    if: github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'release-v2.7'
+    needs: onLabelAndApproval
+    if: needs.onLabelAndApproval.outputs.is_approved == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.PUSH_TOKEN }}
+
+      - name: Set-up Ruby 3.2
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2' # Not needed with a .ruby-version file
 
-      # Need to remove export version once rancher/charts gets the latest version 
-      # of charts-build-script binary.
-
-      - name: Generate Regsync Config
+      - name: Generate RegSync
         run: |
-          export CHARTS_BUILD_SCRIPT_VERSION=v0.4.2 
+          echo ${{ secrets.PUSH_TOKEN }} | gh auth login --with-token
+          gh pr checkout ${{ github.event.pull_request.number }}
+          git config --global user.email "${{ secrets.USER_GITHUB }}"
+          git config --global user.name "rancherbot"
           make pull-scripts
           make regsync
-
+
+      - name: Check for modifications in regsync.yaml
+        id: check_changes
+        run: |
+            git diff --quiet regsync.yaml || echo "::set-output name=changed::true"
+
+      - name: Commit files
+        if: steps.check_changes.outputs.changed == 'true'
+        run: |
+            git add regsync.yaml
+            git commit -m "Updating resync.yaml"
+            git push
+
+      - name: No changes in regsync.yaml
+        if: steps.check_changes.outputs.changed != 'true'
+        run: |
+            echo "regsync.yaml is the same"
+
       - name: Install Regsync
         run: |
-          curl --silent --fail --location --output regsync https://github.com/regclient/regclient/releases/download/v0.4.8/regsync-linux-amd64
+          curl --silent --fail --location --output regsync https://github.com/regclient/regclient/releases/download/v0.5.1/regsync-linux-amd64
           chmod +x regsync
 
       - name: Sync Images to Registry
         run: |
+          export PATH=$PATH:$(pwd)
           head regsync.yaml
-          time ./regsync once --config regsync.yaml
+          ruby ./scripts/regsync-split.rb
+          tree ./split-regsync
+          time find split-regsync -type f -name split-regsync.yaml -print -exec time regsync once --config '{}' ';'
         env:
           REGISTRY_ENDPOINT: ${{ secrets.REGISTRY_ENDPOINT }}
           REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}

.github/workflows/validation-check.yaml

@@ -0,0 +1,69 @@
+name: Validation Check
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  check-reaction:
+    name: Check for positive reaction on bot's latest validation comment
+    if: startsWith(github.event.pull_request.base.ref, 'dev-v') || startsWith(github.event.pull_request.base.ref, 'release-v')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for modifications in specified folders
+        id: check-modifications
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            // Get the list of files modified in the PR
+            const files = await github.rest.pulls.listFiles({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            // Check if any of the modified files are in the specified folders
+            const modifiedSpecifiedFolders = files.data.some(file => file.filename.startsWith('assets/') || file.filename.startsWith('charts/') || file.filename.startsWith('packages/'));
+
+            return modifiedSpecifiedFolders;
+
+      - name: Check for positive reaction on bot's latest validation comment
+        if: steps.check-modifications.outputs.result == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            // Get comments on the PR
+            const comments = await github.rest.issues.listComments({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo
+            });
+
+            // Sort comments based on their creation datetime in descending order
+            const sortedComments = comments.data.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
+
+            // Find the latest validation comment by github-actions[bot]
+            const latestValidationComment = sortedComments.find(comment => comment.user.login === 'github-actions[bot]' && comment.body.startsWith("## Validation steps"));
+
+            if (latestValidationComment) {
+              const reactions = await github.rest.reactions.listForIssueComment({
+                comment_id: latestValidationComment.id,
+                owner: context.repo.owner,
+                repo: context.repo.repo
+              });
+
+              // Check if there's a thumbs-up reaction on the bot's validation comment
+              const thumbsUpReaction = reactions.data.some(reaction => reaction.content === '+1');
+
+              if (thumbsUpReaction) {
+                console.log("The validation comment by github-actions[bot] has the required thumbs-up reaction.");
+              } else {
+                const createdAt = new Date(latestValidationComment.created_at).toLocaleString('en-US', { timeZoneName: 'short' });
+                console.error("Failed Check - Comment Created At:", createdAt);
+                core.setFailed("The latest validation comment by github-actions[bot] does not have the required thumbs-up reaction!");
+              }
+            } else {
+              core.setFailed("No validation comments by github-actions[bot] found.");
+            }

.github/workflows/validation-comment.yaml

@@ -0,0 +1,39 @@
+name: Validation Comment
+
+on:
+  pull_request_target:
+    branches:
+      - dev-v*
+      - release-v*
+    paths:
+      - 'assets/**'
+      - 'charts/**'
+      - 'packages/**'
+
+jobs:
+  validation-comment:
+    name: Make validation comment on PR
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - name: Make validation comment
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body:
+              `## Validation steps
+              - Ensure all container images have repository and tag on the same level to ensure that all container images are included in rancher-images.txt which are used by airgap customers.
+              <pre>
+              Ex:-
+                longhorn-controller:
+                  repository: rancher/hardened-sriov-cni
+                  tag: v2.6.3-build20230913
+              </pre>
+              - Add a 👍 (thumbs up) reaction to this comment once done. CI won't pass without this reaction to the github-action bot's latest validation comment.
+              - Approve the PR to run the CI check.`
+            })

Makefile

@@ -7,7 +7,10 @@ remove:
 forward-port:
 	./scripts/forward-port
 
-TARGETS := prepare patch clean clean-cache charts list index unzip zip standardize validate template regsync
+check-release-yaml:
+	./scripts/check-release-yaml
+
+TARGETS := prepare patch clean clean-cache charts list index unzip zip standardize validate template regsync check-images check-rc
 
 $(TARGETS):
 	@./scripts/pull-scripts