diff --git a/.github/scripts/gcp-cosmian-vm-startup-script.sh b/.github/scripts/gcp-cosmian-vm-startup-script.sh deleted file mode 100755 index d95bc8fc..00000000 --- a/.github/scripts/gcp-cosmian-vm-startup-script.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -ex - -supervisorctl start cosmian_vm_agent diff --git a/.github/scripts/gcp-kms-startup-script.sh b/.github/scripts/gcp-kms-startup-script.sh deleted file mode 100644 index d9fa03b7..00000000 --- a/.github/scripts/gcp-kms-startup-script.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -ex - -supervisorctl start cosmian_vm_agent - -# wait for cert and key to be generated by `cosmian_vm_agent` before starting nginx -until [ -f /var/lib/cosmian_vm/data/cert.pem ]; do sleep 1; done - -if command -v restorecon &>/dev/null; then - chown nginx /var/lib/cosmian_vm/data/*.pem - restorecon /var/lib/cosmian_vm/data/*.pem -else - chown www-data /var/lib/cosmian_vm/data/*.pem -fi diff --git a/.github/scripts/gcp-kms-tests.sh b/.github/scripts/gcp-kms-tests.sh index 9623a881..9289f8cc 100644 --- a/.github/scripts/gcp-kms-tests.sh +++ b/.github/scripts/gcp-kms-tests.sh @@ -19,21 +19,15 @@ echo "Checking Cosmian KMS HTTP connection..." curl "http://${IP_ADDR}:8080/version" echo "" -# Bypass HTTPS connection on RHEL for now -# to make it work, we must apply correct permissions on: -# /usr/sbin/restorecon /var/lib/cosmian_vm/data/cert.pem -# /usr/sbin/restorecon /var/lib/cosmian_vm/data/key.pem -# if [ ! -f /etc/redhat-release ]; then -# echo "[ OK ] Cosmian KMS HTTP connection" -# echo "Checking Cosmian KMS HTTPS connection..." -# curl --insecure "https://${IP_ADDR}/version" -# echo "" -# echo "[ OK ] Cosmian KMS HTTPS connection" -# echo "Checking Cosmian KMS HTTP to HTTPS redirect connection..." -# curl --insecure "http://${IP_ADDR}/version" -# echo "" -# echo "[ OK ] Cosmian KMS HTTP to HTTPS redirect connection" -# fi +echo "[ OK ] Cosmian KMS HTTP connection" +echo "Checking Cosmian KMS HTTPS connection..." +curl --insecure "https://${IP_ADDR}/version" +echo "" +echo "[ OK ] Cosmian KMS HTTPS connection" +echo "Checking Cosmian KMS HTTP to HTTPS redirect connection..." +curl --insecure "http://${IP_ADDR}/version" +echo "" +echo "[ OK ] Cosmian KMS HTTP to HTTPS redirect connection" echo "Rebooting instance..." gcloud "${MODE}" compute instances stop "$CI_INSTANCE" --zone "$ZONE" --project "$GCP_DEV_PROJECT" @@ -57,18 +51,12 @@ echo "Checking Cosmian KMS HTTP connection..." curl "http://${IP_ADDR}:8080/version" echo "" -# Bypass HTTPS connection on RHEL for now -# to make it work, we must apply correct permissions on: -# /usr/sbin/restorecon /var/lib/cosmian_vm/data/cert.pem -# /usr/sbin/restorecon /var/lib/cosmian_vm/data/key.pem -# if [ ! -f /etc/redhat-release ]; then -# echo "[ OK ] Cosmian KMS HTTP connection" -# echo "Checking Cosmian KMS HTTPS connection..." -# curl --insecure "https://${IP_ADDR}/version" -# echo "" -# echo "[ OK ] Cosmian KMS HTTPS connection" -# echo "Checking Cosmian KMS HTTP to HTTPS redirect connection..." -# curl --insecure "http://${IP_ADDR}/version" -# echo "" -# echo "[ OK ] Cosmian KMS HTTP to HTTPS redirect connection" -# fi +echo "[ OK ] Cosmian KMS HTTP connection" +echo "Checking Cosmian KMS HTTPS connection..." +curl --insecure "https://${IP_ADDR}/version" +echo "" +echo "[ OK ] Cosmian KMS HTTPS connection" +echo "Checking Cosmian KMS HTTP to HTTPS redirect connection..." +curl --insecure "http://${IP_ADDR}/version" +echo "" +echo "[ OK ] Cosmian KMS HTTP to HTTPS redirect connection" diff --git a/.github/scripts/packer_build.sh b/.github/scripts/packer_build.sh index 64df1a53..bb9b6f26 100644 --- a/.github/scripts/packer_build.sh +++ b/.github/scripts/packer_build.sh @@ -31,4 +31,6 @@ sed -i "s#TEMPLATE_COSMIAN_KMS_VERSION#${KMS_VERSION}#g" "$PACKER_FILE" cat "$PACKER_FILE" packer init "$PACKER_FILE" -packer build "$PACKER_FILE" + +# Since packer build fails randomly because of external resources use, retry packer buid until it succeeds +timeout 30m bash -c "until packer build $PACKER_FILE; do sleep 30; done" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6167d96..c8ee880b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: push: branches: - '**' - tags: - - '**' name: CI @@ -13,30 +11,3 @@ jobs: build-cosmian-vm-binaries: name: Build binaries uses: ./.github/workflows/build.yml - - build-test-gcp-sev-image: - strategy: - fail-fast: false - matrix: - distrib: [ubuntu, rhel] - product: [cosmian-vm, kms] - name: (GCP) ${{ matrix.product }} - ${{ matrix.distrib }} - SEV - needs: build-cosmian-vm-binaries - secrets: inherit - uses: ./.github/workflows/gcp_image.yml - with: - techno: sev - distrib: ${{ matrix.distrib }} - machine-type: n2d-standard-2 - zone: europe-west4-a - confidential-compute-type: SEV_SNP - min-cpu-platform: AMD Milan - project: cosmian-dev - mode: beta - product: ${{ matrix.product }} - - release: - name: Release - needs: [build-test-gcp-sev-image] - secrets: inherit - uses: ./.github/workflows/release.yml diff --git a/.github/workflows/gcp.yml b/.github/workflows/gcp.yml index c1505abb..fbad6626 100644 --- a/.github/workflows/gcp.yml +++ b/.github/workflows/gcp.yml @@ -26,6 +26,11 @@ on: options: - cosmian-vm - kms + kms-version: + description: Cosmian KMS version (None or X.Y.Z) + required: true + default: X.Y.Z + type: string name: Standalone GCP build @@ -48,8 +53,8 @@ jobs: min-cpu-platform: AMD Milan project: cosmian-dev mode: beta - maintenance: maintenance-policy=TERMINATE product: ${{ inputs.product }} + kms-version: ${{ inputs.kms-version }} build-test-gcp-tdx-image: if: contains(inputs.cpu, 'tdx') @@ -66,5 +71,5 @@ jobs: min-cpu-platform: AUTOMATIC project: cosmian-dev mode: alpha - maintenance: on-host-maintenance=TERMINATE product: ${{ inputs.product }} + kms-version: ${{ inputs.kms-version }} diff --git a/.github/workflows/gcp_image.yml b/.github/workflows/gcp_image.yml index 0b55473a..a4879816 100644 --- a/.github/workflows/gcp_image.yml +++ b/.github/workflows/gcp_image.yml @@ -31,11 +31,13 @@ on: product: required: true type: string + kms-version: + required: true + type: string env: GCP_DEV_PROJECT: cosmian-dev GCP_PUBLIC_PROJECT: cosmian-public - KMS_VERSION: 4.15.0 jobs: build-image: @@ -81,6 +83,7 @@ jobs: env: TIMESTAMP: ${{ steps.env.outputs.TIMESTAMP }} IMAGE_NAME: ${{ steps.env.outputs.IMAGE_NAME }} + KMS_VERSION: ${{ inputs.kms-version }} run: | set -ex COSMIAN_VM_VERSION="last_build/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" @@ -92,6 +95,7 @@ jobs: TIMESTAMP: ${{ steps.env.outputs.TIMESTAMP }} IMAGE_NAME: ${{ steps.env.outputs.IMAGE_NAME }} VERSION: ${{ github.ref_name }} + KMS_VERSION: ${{ inputs.kms-version }} run: | set -ex COSMIAN_VM_VERSION="$VERSION" @@ -137,7 +141,6 @@ jobs: --project ${{ inputs.project }} \ --tags ssh-full,backend,backend-rust-full,http-server,https-server,cosmian-vm-agent,${{ inputs.product }} \ --maintenance-policy=TERMINATE \ - --metadata-from-file=startup-script=.github/scripts/gcp-${{ inputs.product }}-startup-script.sh \ --shielded-secure-boot \ --max-run-duration=20m \ --instance-termination-action=DELETE @@ -209,11 +212,12 @@ jobs: CI_INSTANCE: ${{ needs.build-image.outputs.ci_instance }} IMAGE_NAME: ${{ needs.build-image.outputs.image_name }} TAG: ${{ github.ref_name }} + KMS: ${{ inputs.kms-version }} run: | set -ex VERSION=$(echo $TAG | sed 's/\./-/g; s/_/-/g; s/+/-/g') - KMS_VERSION=$(echo $KMS_VERSION | sed 's/\./-/g; s/_/-/g; s/+/-/g') + KMS_VERSION=$(echo $KMS | sed 's/\./-/g; s/_/-/g; s/+/-/g') NEW_IMAGE_NAME=cosmian-vm-$VERSION-${{ inputs.techno }}-${{ inputs.distrib }} if [ "${{ inputs.distrib }}" = "ubuntu" ]; then diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..ece31b89 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,43 @@ +--- +on: + # Run only on pull requests and tags + push: + tags: + - '*' + pull_request: + +name: Pull requests CI + +jobs: + build-cosmian-vm-binaries: + name: Build binaries + uses: ./.github/workflows/build.yml + + build-test-gcp-sev-image: + strategy: + fail-fast: false + matrix: + distrib: [ubuntu, rhel] + product: [cosmian-vm, kms] + name: (GCP) ${{ matrix.product }} - ${{ matrix.distrib }} - SEV + needs: build-cosmian-vm-binaries + secrets: inherit + uses: ./.github/workflows/gcp_image.yml + with: + techno: sev + distrib: ${{ matrix.distrib }} + machine-type: n2d-standard-2 + zone: europe-west4-a + confidential-compute-type: SEV_SNP + min-cpu-platform: AMD Milan + project: cosmian-dev + mode: beta + product: ${{ matrix.product }} + kms-version: 4.15.0 + + release: + name: Release + if: startsWith(github.ref, 'refs/tags/') + needs: [build-test-gcp-sev-image] + secrets: inherit + uses: ./.github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5684514c..007dc941 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,15 @@ jobs: with: name: cosmian_vm + - name: Release on tags, attach asset on release + uses: softprops/action-gh-release@v1 + with: + files: | + CHANGELOG.md + target/release/cosmian_vm_agent + target/release/cosmian_vm + target/release/cosmian_certtool + cleanup: - if: startsWith(github.ref, 'refs/tags/') uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@main secrets: inherit diff --git a/CHANGELOG.md b/CHANGELOG.md index e8814594..8157d695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,19 @@ All notable changes to this project will be documented in this file. ## [1.1.0] - 2024-04-12 -### Ci +### Features -- Push VM/KMS images on GCP on tags +- For GCP (SEV) ([#94](https://github.com/Cosmian/cosmian_vm/pull/94)): + - Deploy Cosmian VM/KMS images based on `ubuntu-2204-jammy-v20240319` and `rhel-9-v20240312`. Images deployment on tags only. + - Remove use of startup scripts: + - cosmian_vm_agent is auto-restarting on failures + - for KMS, nginx is auto-restarting on failures +- For Azure (SEV): + - Add Ansible Cosmian VM/KMS installation ### Bug Fixes -- Fix reboot problem on RHEL and add KMS installation via Ansible ([#84](https://github.com/Cosmian/cosmian_vm/pull/84)) +- Fix reboot problem on RHEL ([#84](https://github.com/Cosmian/cosmian_vm/pull/84)) ## [1.1.0-rc.4] - 2024-04-05 diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 0940c3a2..6ce49653 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,5 +1,5 @@ --- -cosmian_vm_version: 1.1.0-rc.4 +cosmian_vm_version: 1.1.0 cosmian_kms_version: 4.15.0 nginx_user: root diff --git a/ansible/kms-playbook.yml b/ansible/kms-playbook.yml index df323002..c2db7f61 100644 --- a/ansible/kms-playbook.yml +++ b/ansible/kms-playbook.yml @@ -8,16 +8,7 @@ - cleanup_cosmian_vm_agent - name: Install KMS - when: ansible_distribution == 'Ubuntu' ansible.builtin.import_playbook: kms-packer-playbook.yml - vars: - nginx_user: www-data - -- name: Install KMS - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' or ansible_distribution == 'Rocky' - ansible.builtin.import_playbook: kms-packer-playbook.yml - vars: - nginx_user: nginx - name: Start and check KMS hosts: all diff --git a/ansible/roles/configure_ima/tasks/main.yml b/ansible/roles/configure_ima/tasks/main.yml index fdf1ca87..6751d655 100644 --- a/ansible/roles/configure_ima/tasks/main.yml +++ b/ansible/roles/configure_ima/tasks/main.yml @@ -126,6 +126,7 @@ - name: Selinux - allow https ansible.builtin.command: setsebool -P httpd_can_network_connect 1 + tags: selinux - name: Dracut ansible.builtin.command: dracut --force diff --git a/ansible/roles/cosmian_vm_agent/templates/supervisor_cosmian_vm_agent.conf.j2 b/ansible/roles/cosmian_vm_agent/templates/supervisor_cosmian_vm_agent.conf.j2 index 75fe9dc9..0d1adb21 100644 --- a/ansible/roles/cosmian_vm_agent/templates/supervisor_cosmian_vm_agent.conf.j2 +++ b/ansible/roles/cosmian_vm_agent/templates/supervisor_cosmian_vm_agent.conf.j2 @@ -1,7 +1,7 @@ [program:cosmian_vm_agent] command=cosmian_vm_agent directory=/usr/sbin -autostart=false +autostart=true autorestart=true startretries=100 priority=998 diff --git a/ansible/roles/kms/tasks/main.yml b/ansible/roles/kms/tasks/main.yml index 9f46936a..b1fe540d 100644 --- a/ansible/roles/kms/tasks/main.yml +++ b/ansible/roles/kms/tasks/main.yml @@ -89,17 +89,12 @@ group: root mode: "0644" -- name: Add Nginx configuration - when: ansible_distribution == 'Ubuntu' - ansible.builtin.template: - src: nginx.j2 - dest: /etc/nginx/nginx.conf - owner: root - group: root - mode: "0644" +- name: Selinux - add nginx to permissive allow httpd + when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' or ansible_distribution == 'Rocky' + ansible.builtin.command: semanage permissive -a httpd_t + tags: selinux - name: Add Nginx configuration - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' or ansible_distribution == 'Rocky' ansible.builtin.template: src: nginx.j2 dest: /etc/nginx/nginx.conf @@ -107,8 +102,14 @@ group: root mode: "0644" +- name: Enable systemd Nginx service + ansible.builtin.systemd_service: + name: nginx + enabled: true + state: stopped + daemon_reload: true + - name: Make systemd Nginx restarting on failure - when: ansible_distribution == 'Ubuntu' ansible.builtin.blockinfile: path: /etc/systemd/system/multi-user.target.wants/nginx.service block: | @@ -117,15 +118,9 @@ RestartSec=10s tags: systemd_nginx -- name: Enable systemd Nginx service - ansible.builtin.systemd_service: - name: nginx - enabled: true - state: stopped - daemon_reload: true - - name: Enable systemd Redis service ansible.builtin.systemd_service: name: redis enabled: true state: stopped + daemon_reload: true diff --git a/ansible/roles/start_cosmian_vm/tasks/main.yml b/ansible/roles/start_cosmian_vm/tasks/main.yml index 460f9fb2..78a595ad 100644 --- a/ansible/roles/start_cosmian_vm/tasks/main.yml +++ b/ansible/roles/start_cosmian_vm/tasks/main.yml @@ -15,22 +15,6 @@ state: started tags: launch -- name: Override Cosmian VM supervisor configuration with autostart to true - when: ansible_distribution == 'Ubuntu' - ansible.builtin.lineinfile: - path: /etc/supervisor/conf.d/cosmian_vm_agent.conf - regexp: autostart=false - line: autostart=true - backup: true - -- name: Override Cosmian VM supervisor configuration with autostart to true - when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' or ansible_distribution == 'Rocky' - ansible.builtin.lineinfile: - path: /etc/supervisord.d/cosmian_vm_agent.ini - regexp: autostart=false - line: autostart=true - backup: true - - name: Start Cosmian VM Agent community.general.supervisorctl: name: cosmian_vm_agent diff --git a/ansible/roles/start_kms/tasks/main.yml b/ansible/roles/start_kms/tasks/main.yml index af6ae94c..0dac3d52 100644 --- a/ansible/roles/start_kms/tasks/main.yml +++ b/ansible/roles/start_kms/tasks/main.yml @@ -57,25 +57,10 @@ - cosmian_vm_output.rc != 0 tags: app_init -- name: Override KMS supervisor configuration with autostart to true - when: ansible_distribution == 'Ubuntu' - ansible.builtin.lineinfile: - path: /etc/supervisor/conf.d/cosmian_kms.conf - regexp: autostart=false - line: autostart=true - backup: true - - name: Override KMS supervisor configuration with autostart to true when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' or ansible_distribution == 'Rocky' tags: certificate_nginx_permissions block: - - name: Override KMS supervisor configuration with autostart to true - ansible.builtin.lineinfile: - path: /etc/supervisord.d/cosmian_kms.ini - regexp: autostart=false - line: autostart=true - backup: true - - name: Restorecon on cert and key ansible.builtin.command: cmd: | diff --git a/packer/gcp.pkr.hcl b/packer/gcp.pkr.hcl index 68fc675d..e367a741 100644 --- a/packer/gcp.pkr.hcl +++ b/packer/gcp.pkr.hcl @@ -15,7 +15,7 @@ variable "ssh_username" { variable "ssh_timeout" { type = string - default = "20m" + default = "5m" } variable "image_guest_os_features" { @@ -45,7 +45,7 @@ variable "use_os_login" { variable "wait_to_add_ssh_keys" { type = string - default = "30s" + default = "60s" } source "googlecompute" "TEMPLATE_GOOGLE_COMPUTE" {