From abb8cfc60781d97053f367e58c5b928f83a318b6 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 8 Apr 2024 11:13:02 +0200 Subject: [PATCH 1/7] Integrate FIPS stemcell update into update-releases pipeline * use refactored update-stemcell-ops task to determine stemcell version * new ops file "operations/fips-stemcell.yml" to maintain version and as trigger for cf-deployment pipeline --- ci/pipelines/update-releases.yml | 52 ++++++++++++++++++++++++++++ ci/template/update-releases.yml | 59 ++++++++++++++++++++++++++++++++ operations/fips-stemcell.yml | 5 +++ 3 files changed, 116 insertions(+) create mode 100644 operations/fips-stemcell.yml diff --git a/ci/pipelines/update-releases.yml b/ci/pipelines/update-releases.yml index 4ae09c828..8a1139d55 100644 --- a/ci/pipelines/update-releases.yml +++ b/ci/pipelines/update-releases.yml @@ -119,6 +119,9 @@ groups: jobs: - update-windows2019-stemcell - update-windows2019fs-offline-release +- name: update-fips-stemcell + jobs: + - update-fips-stemcell - name: debug jobs: [] - name: cleanup @@ -143,6 +146,11 @@ resource_types: source: repository: cfcommunity/slack-notification-resource tag: latest +- name: bosh-io-stemcell + source: + repository: foundationalinfrastructure/bosh-io-stemcell-resource + tag: v1.2.1 + type: docker-image resources: - name: cf-deployment-all-branches type: git @@ -170,6 +178,13 @@ resources: branch: main uri: git@github.com:cloudfoundry/cf-deployment.git private_key: ((ard_wg_gitbot_ssh_key.private_key)) +- name: cf-deployment-fips + type: git + icon: github + source: + branch: integrate_fips_validation + uri: git@github.com:cloudfoundry/cf-deployment.git + private_key: ((ard_wg_gitbot_ssh_key.private_key)) - name: cf-deployment-version type: semver source: @@ -208,11 +223,25 @@ resources: icon: github source: uri: https://github.com/cloudfoundry/runtime-ci.git +- name: runtime-ci-fips-branch + type: git + icon: github + source: + uri: https://github.com/cloudfoundry/runtime-ci.git + branch: refactor_update_stemcell_task - name: stemcell type: bosh-io-stemcell icon: dna source: name: bosh-google-kvm-ubuntu-jammy-go_agent +- name: fips-stemcell + type: bosh-io-stemcell + icon: dna + source: + name: bosh-aws-xen-hvm-ubuntu-jammy-fips-go_agent + auth: + access_key: ((ci_dev_gcp_service_account_hmac_access_key)) + secret_key: ((ci_dev_gcp_service_account_hmac_secret)) - name: stemcell-version-bump-detect type: stemcell-version-bump icon: dna @@ -14029,6 +14058,29 @@ jobs: params: rebase: true repository: updated-stemcell-ops-file +- name: update-fips-stemcell + public: true + serial: true + plan: + - in_parallel: + - get: runtime-ci-fips-branch + - get: cf-deployment-fips + - get: fips-stemcell + trigger: true + params: + tarball: false + - task: update-stemcell-ops + file: runtime-ci/tasks/update-stemcell-ops/task.yml + input_mapping: + ops-files: cf-deployment-develop + stemcell: fips-stemcell + params: + ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml + UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml + - put: cf-deployment-fips + params: + rebase: true + repository: updated-stemcell-ops-file - name: detect-stemcell-bump plan: - in_parallel: diff --git a/ci/template/update-releases.yml b/ci/template/update-releases.yml index 38a3f65b9..2c20ebcfe 100644 --- a/ci/template/update-releases.yml +++ b/ci/template/update-releases.yml @@ -53,6 +53,9 @@ groups: #@ for r in data.values.windowsOfflineReleases: - #@ "update-" + r.name + "-offline-release" #@ end +- name: update-fips-stemcell + jobs: + - update-fips-stemcell - name: debug jobs: #@ for r in data.values.baseReleases: @@ -103,6 +106,13 @@ resource_types: repository: cfcommunity/slack-notification-resource tag: latest +#! TODO remove this resource type declaration when a final release of the resource is available +- name: bosh-io-stemcell + source: + repository: foundationalinfrastructure/bosh-io-stemcell-resource + tag: v1.2.1 + type: docker-image + resources: - name: cf-deployment-all-branches type: git @@ -134,6 +144,14 @@ resources: uri: git@github.com:cloudfoundry/cf-deployment.git private_key: ((ard_wg_gitbot_ssh_key.private_key)) +- name: cf-deployment-fips + type: git + icon: github + source: + branch: integrate_fips_validation + uri: git@github.com:cloudfoundry/cf-deployment.git + private_key: ((ard_wg_gitbot_ssh_key.private_key)) + - name: cf-deployment-version type: semver source: @@ -178,12 +196,28 @@ resources: source: uri: https://github.com/cloudfoundry/runtime-ci.git +- name: runtime-ci-fips-branch + type: git + icon: github + source: + uri: https://github.com/cloudfoundry/runtime-ci.git + branch: refactor_update_stemcell_task + - name: stemcell type: bosh-io-stemcell icon: dna source: name: bosh-google-kvm-ubuntu-jammy-go_agent +- name: fips-stemcell + type: bosh-io-stemcell + icon: dna + source: + name: bosh-aws-xen-hvm-ubuntu-jammy-fips-go_agent + auth: + access_key: ((ci_dev_gcp_service_account_hmac_access_key)) + secret_key: ((ci_dev_gcp_service_account_hmac_secret)) + - name: stemcell-version-bump-detect type: stemcell-version-bump icon: dna @@ -727,6 +761,31 @@ jobs: repository: updated-stemcell-ops-file #@ end +- name: update-fips-stemcell + public: true + serial: true + plan: + - in_parallel: + - get: runtime-ci-fips-branch + - get: cf-deployment-fips + - get: fips-stemcell + trigger: true + params: + tarball: false + - task: update-stemcell-ops + file: runtime-ci/tasks/update-stemcell-ops/task.yml + input_mapping: + ops-files: cf-deployment-develop + stemcell: fips-stemcell + params: + STEMCELL_STACK: ubuntu-jammy + ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml + UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml + - put: cf-deployment-fips + params: + rebase: true + repository: updated-stemcell-ops-file + - name: detect-stemcell-bump plan: - in_parallel: diff --git a/operations/fips-stemcell.yml b/operations/fips-stemcell.yml new file mode 100644 index 000000000..64bcd20cc --- /dev/null +++ b/operations/fips-stemcell.yml @@ -0,0 +1,5 @@ +- type: replace + path: /stemcells/- + value: + alias: ubuntu-jammy + version: "1.406" \ No newline at end of file From 88db4a1752402edda985aa27fa5410e2b8d1038e Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 8 Apr 2024 11:27:48 +0200 Subject: [PATCH 2/7] manually reset version to test update job --- operations/fips-stemcell.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/operations/fips-stemcell.yml b/operations/fips-stemcell.yml index 64bcd20cc..b49316b74 100644 --- a/operations/fips-stemcell.yml +++ b/operations/fips-stemcell.yml @@ -2,4 +2,5 @@ path: /stemcells/- value: alias: ubuntu-jammy - version: "1.406" \ No newline at end of file + os: ubuntu-jammy + version: "1.406" From 7212111f71ae663ed802a2c115448079e14e3218 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 8 Apr 2024 11:29:41 +0200 Subject: [PATCH 3/7] fix update-fips jobs --- ci/pipelines/update-releases.yml | 6 ++++-- ci/template/update-releases.yml | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/pipelines/update-releases.yml b/ci/pipelines/update-releases.yml index 8a1139d55..a4a40457d 100644 --- a/ci/pipelines/update-releases.yml +++ b/ci/pipelines/update-releases.yml @@ -14070,11 +14070,13 @@ jobs: params: tarball: false - task: update-stemcell-ops - file: runtime-ci/tasks/update-stemcell-ops/task.yml + file: runtime-ci-fips-branch/tasks/update-stemcell-ops/task.yml input_mapping: - ops-files: cf-deployment-develop + runtime-ci: runtime-ci-fips-branch + ops-files: cf-deployment-fips stemcell: fips-stemcell params: + STEMCELL_STACK: ubuntu-jammy ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml - put: cf-deployment-fips diff --git a/ci/template/update-releases.yml b/ci/template/update-releases.yml index 2c20ebcfe..05e07c590 100644 --- a/ci/template/update-releases.yml +++ b/ci/template/update-releases.yml @@ -773,9 +773,10 @@ jobs: params: tarball: false - task: update-stemcell-ops - file: runtime-ci/tasks/update-stemcell-ops/task.yml + file: runtime-ci-fips-branch/tasks/update-stemcell-ops/task.yml input_mapping: - ops-files: cf-deployment-develop + runtime-ci: runtime-ci-fips-branch + ops-files: cf-deployment-fips stemcell: fips-stemcell params: STEMCELL_STACK: ubuntu-jammy From 3c450ead655edf1d4a4c96678a5d39959a847766 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 8 Apr 2024 11:31:22 +0200 Subject: [PATCH 4/7] reset stemcell alias to "default" --- operations/fips-stemcell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/fips-stemcell.yml b/operations/fips-stemcell.yml index b49316b74..9c65ba3f7 100644 --- a/operations/fips-stemcell.yml +++ b/operations/fips-stemcell.yml @@ -1,6 +1,6 @@ - type: replace path: /stemcells/- value: - alias: ubuntu-jammy + alias: default os: ubuntu-jammy version: "1.406" From 6f63db1a64dc57c1c2024f4a6d2709a44220a427 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Mon, 8 Apr 2024 13:00:59 +0200 Subject: [PATCH 5/7] remove dev branch references --- ci/input/inputs.yml | 1 + ci/pipelines/update-releases.yml | 33 +++++++++--------------------- ci/template/update-releases.yml | 35 +++++++++----------------------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/ci/input/inputs.yml b/ci/input/inputs.yml index 300d5675c..e8bf642df 100644 --- a/ci/input/inputs.yml +++ b/ci/input/inputs.yml @@ -131,6 +131,7 @@ untestedOpsReleases: windowsStemcells: - name: windows2019 + stack: windows2019 opsFile: windows2019-cell.yml opsFileDir: operations diff --git a/ci/pipelines/update-releases.yml b/ci/pipelines/update-releases.yml index a4a40457d..b8634afe7 100644 --- a/ci/pipelines/update-releases.yml +++ b/ci/pipelines/update-releases.yml @@ -178,13 +178,6 @@ resources: branch: main uri: git@github.com:cloudfoundry/cf-deployment.git private_key: ((ard_wg_gitbot_ssh_key.private_key)) -- name: cf-deployment-fips - type: git - icon: github - source: - branch: integrate_fips_validation - uri: git@github.com:cloudfoundry/cf-deployment.git - private_key: ((ard_wg_gitbot_ssh_key.private_key)) - name: cf-deployment-version type: semver source: @@ -223,12 +216,6 @@ resources: icon: github source: uri: https://github.com/cloudfoundry/runtime-ci.git -- name: runtime-ci-fips-branch - type: git - icon: github - source: - uri: https://github.com/cloudfoundry/runtime-ci.git - branch: refactor_update_stemcell_task - name: stemcell type: bosh-io-stemcell icon: dna @@ -14047,13 +14034,14 @@ jobs: params: tarball: false - task: update-windows-stemcell-ops - file: runtime-ci/tasks/update-windows-stemcell-ops/task.yml + file: runtime-ci/tasks/update-stemcell-ops/task.yml input_mapping: ops-files: cf-deployment-develop - windows-stemcell: windows2019-stemcell + stemcell: windows2019-stemcell params: - ORIGINAL_WINDOWS_OPS_FILE_PATH: operations/windows2019-cell.yml - UPDATED_WINDOWS_OPS_FILE_PATH: operations/windows2019-cell.yml + STEMCELL_STACK: windows2019 + ORIGINAL_OPS_FILE_PATH: operations/windows2019-cell.yml + UPDATED_OPS_FILE_PATH: operations/windows2019-cell.yml - put: cf-deployment-develop params: rebase: true @@ -14063,23 +14051,22 @@ jobs: serial: true plan: - in_parallel: - - get: runtime-ci-fips-branch - - get: cf-deployment-fips + - get: runtime-ci + - get: cf-deployment-develop - get: fips-stemcell trigger: true params: tarball: false - task: update-stemcell-ops - file: runtime-ci-fips-branch/tasks/update-stemcell-ops/task.yml + file: runtime-ci/tasks/update-stemcell-ops/task.yml input_mapping: - runtime-ci: runtime-ci-fips-branch - ops-files: cf-deployment-fips + ops-files: cf-deployment-develop stemcell: fips-stemcell params: STEMCELL_STACK: ubuntu-jammy ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml - - put: cf-deployment-fips + - put: cf-deployment-develop params: rebase: true repository: updated-stemcell-ops-file diff --git a/ci/template/update-releases.yml b/ci/template/update-releases.yml index 05e07c590..d7eb6b662 100644 --- a/ci/template/update-releases.yml +++ b/ci/template/update-releases.yml @@ -144,14 +144,6 @@ resources: uri: git@github.com:cloudfoundry/cf-deployment.git private_key: ((ard_wg_gitbot_ssh_key.private_key)) -- name: cf-deployment-fips - type: git - icon: github - source: - branch: integrate_fips_validation - uri: git@github.com:cloudfoundry/cf-deployment.git - private_key: ((ard_wg_gitbot_ssh_key.private_key)) - - name: cf-deployment-version type: semver source: @@ -196,13 +188,6 @@ resources: source: uri: https://github.com/cloudfoundry/runtime-ci.git -- name: runtime-ci-fips-branch - type: git - icon: github - source: - uri: https://github.com/cloudfoundry/runtime-ci.git - branch: refactor_update_stemcell_task - - name: stemcell type: bosh-io-stemcell icon: dna @@ -748,13 +733,14 @@ jobs: params: tarball: false - task: update-windows-stemcell-ops - file: runtime-ci/tasks/update-windows-stemcell-ops/task.yml + file: runtime-ci/tasks/update-stemcell-ops/task.yml input_mapping: ops-files: cf-deployment-develop - windows-stemcell: #@ s.name + "-stemcell" + stemcell: #@ s.name + "-stemcell" params: - ORIGINAL_WINDOWS_OPS_FILE_PATH: #@ "{}/{}".format(s.opsFileDir, s.opsFile) - UPDATED_WINDOWS_OPS_FILE_PATH: #@ "{}/{}".format(s.opsFileDir, s.opsFile) + STEMCELL_STACK: #@ s.stack + ORIGINAL_OPS_FILE_PATH: #@ "{}/{}".format(s.opsFileDir, s.opsFile) + UPDATED_OPS_FILE_PATH: #@ "{}/{}".format(s.opsFileDir, s.opsFile) - put: cf-deployment-develop params: rebase: true @@ -766,23 +752,22 @@ jobs: serial: true plan: - in_parallel: - - get: runtime-ci-fips-branch - - get: cf-deployment-fips + - get: runtime-ci + - get: cf-deployment-develop - get: fips-stemcell trigger: true params: tarball: false - task: update-stemcell-ops - file: runtime-ci-fips-branch/tasks/update-stemcell-ops/task.yml + file: runtime-ci/tasks/update-stemcell-ops/task.yml input_mapping: - runtime-ci: runtime-ci-fips-branch - ops-files: cf-deployment-fips + ops-files: cf-deployment-develop stemcell: fips-stemcell params: STEMCELL_STACK: ubuntu-jammy ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml - - put: cf-deployment-fips + - put: cf-deployment-develop params: rebase: true repository: updated-stemcell-ops-file From a21f33ee7dc784878c98a6363e5c2a8e576dd46e Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Thu, 25 Apr 2024 16:00:40 +0200 Subject: [PATCH 6/7] Move fips-stemcell ops file to "operations/test" folder --- ci/pipelines/update-releases.yml | 4 ++-- ci/template/update-releases.yml | 4 ++-- operations/test/README.md | 1 + operations/{ => test}/fips-stemcell.yml | 0 4 files changed, 5 insertions(+), 4 deletions(-) rename operations/{ => test}/fips-stemcell.yml (100%) diff --git a/ci/pipelines/update-releases.yml b/ci/pipelines/update-releases.yml index b8634afe7..75adf8282 100644 --- a/ci/pipelines/update-releases.yml +++ b/ci/pipelines/update-releases.yml @@ -14064,8 +14064,8 @@ jobs: stemcell: fips-stemcell params: STEMCELL_STACK: ubuntu-jammy - ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml - UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml + ORIGINAL_OPS_FILE_PATH: operations/test/fips-stemcell.yml + UPDATED_OPS_FILE_PATH: operations/test/fips-stemcell.yml - put: cf-deployment-develop params: rebase: true diff --git a/ci/template/update-releases.yml b/ci/template/update-releases.yml index d7eb6b662..2478fdda1 100644 --- a/ci/template/update-releases.yml +++ b/ci/template/update-releases.yml @@ -765,8 +765,8 @@ jobs: stemcell: fips-stemcell params: STEMCELL_STACK: ubuntu-jammy - ORIGINAL_OPS_FILE_PATH: operations/fips-stemcell.yml - UPDATED_OPS_FILE_PATH: operations/fips-stemcell.yml + ORIGINAL_OPS_FILE_PATH: operations/test/fips-stemcell.yml + UPDATED_OPS_FILE_PATH: operations/test/fips-stemcell.yml - put: cf-deployment-develop params: rebase: true diff --git a/operations/test/README.md b/operations/test/README.md index 0e3d1942a..58c79621c 100644 --- a/operations/test/README.md +++ b/operations/test/README.md @@ -18,5 +18,6 @@ They may change without notice. | [`enable-nfs-test-server.yml`](enable-nfs-test-server.yml) | adds an NFS server to the deployment | nfstestserver can be reached at nfstestserver.service.cf.internal for acceptance testing purposes | | [`enable-nfs-test-ldapserver.yml`](enable-nfs-test-ldapserver.yml) | Adds an LDAP server to the deployment to allow testing of NFS volume services configured with LDAP authentication | Requires enable-nfs-volume-service.yml and enable-nfs-test-server.yml. nfstestldapserver can be reached at nfstestldapserver.service.cf.internal | | [`enable-smb-test-server.yml`](enable-smb-test-server.yml) | adds an SMB server to the deployment | smbtestserver can be reached at smbtestserver.service.cf.internal for acceptance testing purposes | +| [`fips-stemcell.yml`](fips-stemcell.yml) | Contains the validated version of the FIPS-compliant stemcell | | [`speed-up-dynamic-asgs.yml`](speed-up-dynamic-asgs.yml) | decreases the polling time for policy-server-asg-syncer and vxlan-policy-agent to speed up cf-acceptance-tests | Not suitable for production envs | | [`set-smoke-test-timeout-scale.yml`](set-smoke-test-timeout-scale.yml) | set the timeout scale to 5 | used when retrieving logs in the smoke tests timeout. usualy happens with gcp enviorments that do not have a ephemeral ips | \ No newline at end of file diff --git a/operations/fips-stemcell.yml b/operations/test/fips-stemcell.yml similarity index 100% rename from operations/fips-stemcell.yml rename to operations/test/fips-stemcell.yml From 7ff20684c99b76db4bd0e213da19441e3daf0e14 Mon Sep 17 00:00:00 2001 From: Jochen Ehret Date: Fri, 26 Apr 2024 08:11:29 +0200 Subject: [PATCH 7/7] Add unit test for "fips-stemcell.yml" ops file --- units/tests/test_test/operations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/units/tests/test_test/operations.yml b/units/tests/test_test/operations.yml index b4e71d8d4..12d8c16d8 100644 --- a/units/tests/test_test/operations.yml +++ b/units/tests/test_test/operations.yml @@ -19,6 +19,7 @@ enable-smb-test-server.yml: vars: - smb-password=FOO.PASS - smb-username=BAR.USER +fips-stemcell.yml: {} scale-to-one-az-addon-parallel-cats.yml: ops: - ../scale-to-one-az.yml