From 3a4b0fd3159a9a6a2a5008fb00a48a8ac9377970 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 18 Jul 2024 11:41:34 +0100 Subject: [PATCH 1/5] Updated pavi_shared_aws_infra hash ref to include all files affecting package hash value --- shared_aws_infra/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_aws_infra/Makefile b/shared_aws_infra/Makefile index e7aa6b23..e24009b5 100644 --- a/shared_aws_infra/Makefile +++ b/shared_aws_infra/Makefile @@ -11,7 +11,7 @@ build: # Setting SOURCE_DATE_EPOCH to commit date, # so that build produces whl with reproducable hash given same source-code, # resulting in reproducable builds that don't result in hash mismatches on rebuild - REF=$$(git log -n 1 --no-merges --format=%H ./pavi_shared_aws_infra) && \ + REF=$$(git log -n 1 --no-merges --format=%H -- ./pavi_shared_aws_infra pyproject.toml) && \ echo "REF: $$REF" && \ export SOURCE_DATE_EPOCH=$$(git show --no-patch --format=%ct $$REF) && \ echo "SOURCE_DATE_EPOCH: $$SOURCE_DATE_EPOCH" && \ From d858111ce43e8704f1c16cf168fb50d38fb9ce16 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 18 Jul 2024 13:02:34 +0100 Subject: [PATCH 2/5] Removed post-merge lock file pavi_shared_aws_infra hash updates pavi_shared_aws_infra hash should be independent of any requirements.txt updates, and only depend on pyproject.toml file (and src files). --- .github/workflows/PR-validation.yml | 1 - .github/workflows/main-build-and-deploy.yml | 82 +++++++++------------ 2 files changed, 34 insertions(+), 49 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index aaba5be1..92c8e10a 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -1086,7 +1086,6 @@ jobs: echo 'The validation workflow automatically applied the below dependency lock file updates.' cat diff-files.txt | xargs -I {} cat diff-files/{}.md echo '' - echo 'Note that hashes for the pavi_share_aws_infra package can change after merge, as the package hash is commit-date dependent.' echo 'A bundle with all dependency lock files is available [here](${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}) as artifact called `deps_lock_files_bundle`.' } > git-diff-message.md - name: Generate and store no-change message diff --git a/.github/workflows/main-build-and-deploy.yml b/.github/workflows/main-build-and-deploy.yml index 9015a9ff..b9988e61 100644 --- a/.github/workflows/main-build-and-deploy.yml +++ b/.github/workflows/main-build-and-deploy.yml @@ -79,10 +79,8 @@ jobs: - name: Debug printing (git history graph) run: | git log --graph --oneline - # Independent shared_aws_infra lock files commit required to pin hash representing the package - # to be included in depending aws_infra components' lock files. - - name: commit shared_aws_infra dependency lock file changes - id: shared-aws-infra-deps-lock-commit + - name: commit dependency lock file changes + id: deps-lock-commit if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} uses: stefanzweifel/git-auto-commit-action@v5 with: @@ -90,16 +88,37 @@ jobs: commit_user_name: ${{ steps.app-token.outputs.app-slug }}[bot] commit_user_email: ${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com commit_author: ${{ steps.app-token.outputs.app-slug }}[bot] <${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com> - commit_message: Auto-updated pavi_shared_aws_infra deps lock files [skip actions] - file_pattern: 'shared_aws_infra/requirements.txt shared_aws_infra/tests/requirements.txt' + commit_message: Auto-updated deps lock files [skip actions] + file_pattern: '*requirements.txt *package-lock.json' disable_globbing: true skip_checkout: true skip_fetch: true - name: Store commit sha in env variable (if updated) - if: steps.shared-aws-infra-deps-lock-commit.outputs.commit_hash + if: steps.deps-lock-commit.outputs.commit_hash + run: | + echo "latest_commit_sha=${{ steps.deps-lock-commit.outputs.commit_hash }}" >> "$GITHUB_ENV" + - name: store latest_commit_sha job output + id: store-output + run: | + echo "latest-commit-sha=${{ env.latest_commit_sha }}" >> "$GITHUB_OUTPUT" + - name: Debug printing (git status) + run: | + git status + - name: Debug printing (git history graph) run: | - echo "latest_commit_sha=${{ steps.shared-aws-infra-deps-lock-commit.outputs.commit_hash }}" >> "$GITHUB_ENV" - # Build pavi_shared_aws_infra package (to ensure hash includes latest commit date) + git log --graph --oneline + build-pavi-shared-aws-infra: + name: Build pavi_shared_aws_infra package + runs-on: ubuntu-22.04 + needs: [commit-deps-lock-updates, on-deploy] + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + ref: ${{ needs.commit-deps-lock-updates.outputs.latest-commit-sha }} + fetch-depth: 0 + sparse-checkout: | + shared_aws_infra/ - name: Setup Python uses: actions/setup-python@v5 with: @@ -110,53 +129,19 @@ jobs: - name: Debug printing (git history graph) run: | git log --graph --oneline - - name: Build and install the pavi_shared_aws_infra package + - name: Build the pavi_shared_aws_infra package working-directory: ./shared_aws_infra run: | - make clean build install + make clean build - name: Upload package as artifact id: shared-aws-infra-package uses: actions/upload-artifact@v4 with: name: shared_aws_infra_package path: shared_aws_infra/dist/pavi_shared_aws_infra-0.0.0-py3-none-any.whl - - name: Update pavi_shared_aws_infra dependencies - if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} - run: | - make -C api/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only - make -C pipeline/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only - make -C webui/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only - - name: commit remaining dependency lock file changes - id: remaining-aws-infra-deps-lock-commit - if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }} - uses: stefanzweifel/git-auto-commit-action@v5 - with: - branch: ${{ github.base_ref }} - commit_user_name: ${{ steps.app-token.outputs.app-slug }}[bot] - commit_user_email: ${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com - commit_author: ${{ steps.app-token.outputs.app-slug }}[bot] <${{ steps.app-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com> - commit_message: Auto-updated deps lock files [skip actions] - file_pattern: '*requirements.txt *package-lock.json' - disable_globbing: true - skip_checkout: true - skip_fetch: true - - name: Store commit sha in env variable (if updated) - if: steps.remaining-aws-infra-deps-lock-commit.outputs.commit_hash - run: | - echo "latest_commit_sha=${{ steps.remaining-aws-infra-deps-lock-commit.outputs.commit_hash }}" >> "$GITHUB_ENV" - - name: store latest_commit_sha job output - id: store-output - run: | - echo "latest-commit-sha=${{ env.latest_commit_sha }}" >> "$GITHUB_OUTPUT" - - name: Debug printing (git status) - run: | - git status - - name: Debug printing (git history graph) - run: | - git log --graph --oneline pipeline-deploy-aws-infra: name: Deploy/update AWS infrastructure for pipeline - needs: [commit-deps-lock-updates, on-deploy] + needs: [commit-deps-lock-updates, on-deploy, build-pavi-shared-aws-infra] permissions: id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions runs-on: ubuntu-22.04 @@ -197,7 +182,7 @@ jobs: run: make deploy ADD_CDK_ARGS="--require-approval never" api-deploy-image-repo: name: Deploy/update container image repository stack for API - needs: [commit-deps-lock-updates, on-deploy] + needs: [commit-deps-lock-updates, on-deploy, build-pavi-shared-aws-infra] permissions: id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions runs-on: ubuntu-22.04 @@ -241,7 +226,7 @@ jobs: run: make deploy-image-stack ADD_CDK_ARGS="--require-approval never" webui-deploy-image-repo: name: Deploy/update container image repository stack for web UI - needs: [commit-deps-lock-updates, on-deploy] + needs: [commit-deps-lock-updates, on-deploy, build-pavi-shared-aws-infra] permissions: id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions runs-on: ubuntu-22.04 @@ -446,6 +431,7 @@ jobs: needs: - on-deploy - commit-deps-lock-updates + - build-pavi-shared-aws-infra - api-build-and-push-docker-image - pipeline-alignment-build-and-push-docker-image - pipeline-seq-retrieval-build-and-push-docker-image From 1682623e175d78c96219d0742348bc0412e5fdaf Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 18 Jul 2024 14:57:17 +0100 Subject: [PATCH 3/5] Pin setuptools version in pyproject.toml to obtain consistent package hash when built after new setuptools releases --- shared_aws_infra/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared_aws_infra/pyproject.toml b/shared_aws_infra/pyproject.toml index 0642e016..16cc3374 100644 --- a/shared_aws_infra/pyproject.toml +++ b/shared_aws_infra/pyproject.toml @@ -25,5 +25,5 @@ test = [ ] [build-system] -requires = ["setuptools >= 61.0"] +requires = ["setuptools == 71.0.0"] build-backend = "setuptools.build_meta" From 7565d76ec5e535a775ae64f161ee37850a726c5c Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 18 Jul 2024 14:58:15 +0100 Subject: [PATCH 4/5] Avoid failures on temp files outside of pavi_shared_aws_infra directory --- shared_aws_infra/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared_aws_infra/pyproject.toml b/shared_aws_infra/pyproject.toml index 16cc3374..a2fe5972 100644 --- a/shared_aws_infra/pyproject.toml +++ b/shared_aws_infra/pyproject.toml @@ -27,3 +27,5 @@ test = [ [build-system] requires = ["setuptools == 71.0.0"] build-backend = "setuptools.build_meta" +[tool.setuptools] +packages = ["pavi_shared_aws_infra"] From 0bc63a3a32b8481c4947f697f117132185b688aa Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 18 Jul 2024 15:09:26 +0100 Subject: [PATCH 5/5] Revert 7565d76e to ensure subpackages are included --- shared_aws_infra/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared_aws_infra/pyproject.toml b/shared_aws_infra/pyproject.toml index a2fe5972..16cc3374 100644 --- a/shared_aws_infra/pyproject.toml +++ b/shared_aws_infra/pyproject.toml @@ -27,5 +27,3 @@ test = [ [build-system] requires = ["setuptools == 71.0.0"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = ["pavi_shared_aws_infra"]