From 5a46b062e7a5822ff6bc27ff7a2fe2e5de5b88d2 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Wed, 26 Jun 2024 16:01:58 +0200 Subject: [PATCH] Add another e2e workflow that gets run on self-hosted-github runner This is necessary as we want some e2e jobs to run on hosted runners and some to run on self-hosted runners. The actions runner controller does not support using multiple labels to target the runners See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names --- .github/actionlint.yaml | 7 ++ .github/workflows/checks.yml | 40 ++++++++++- .github/workflows/e2e-custom-runner.yml | 90 +++++++++++++++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 4 +- devbox.json | 1 + devbox.lock | 48 +++++++++++++ 6 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 .github/actionlint.yaml create mode 100644 .github/workflows/e2e-custom-runner.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..8c04ab7e5 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - self-hosted-ncn-dind diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80e5471b9..2d6d9a642 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,6 @@ jobs: - Docker # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: @@ -72,6 +71,25 @@ jobs: contents: read checks: write + # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners. + # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names + e2e-quick-start-custom-runner: + strategy: + matrix: + provider: + - Nutanix + # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners. + # - Vsphere (example) + fail-fast: false + uses: ./.github/workflows/e2e-custom-runner.yml + with: + provider: ${{ matrix.provider }} + focus: Quick start + secrets: inherit + permissions: + contents: read + checks: write + e2e-self-hosted: strategy: matrix: @@ -79,7 +97,6 @@ jobs: - Docker # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: @@ -90,6 +107,25 @@ jobs: contents: read checks: write + # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners. + # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names + e2e-self-hosted-custom-runner: + strategy: + matrix: + provider: + - Nutanix + # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners. + # - Vsphere (example) + fail-fast: false + uses: ./.github/workflows/e2e-custom-runner.yml + with: + provider: ${{ matrix.provider }} + focus: Self-hosted + secrets: inherit + permissions: + contents: read + checks: write + lint-go: runs-on: ubuntu-22.04 strategy: diff --git a/.github/workflows/e2e-custom-runner.yml b/.github/workflows/e2e-custom-runner.yml new file mode 100644 index 000000000..d56259338 --- /dev/null +++ b/.github/workflows/e2e-custom-runner.yml @@ -0,0 +1,90 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +on: + workflow_call: + inputs: + provider: + description: Infrastructure provider to run e2e tests with + type: string + required: true + skip: + description: e2e tests to skip + type: string + focus: + description: e2e tests to focus + type: string + +jobs: + e2e-test: + runs-on: "self-hosted-ncn-dind" + permissions: + contents: read + checks: write + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: cachix/install-nix-action@V27 + if: inputs.provider == 'Nutanix' + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Devbox + uses: jetify-com/devbox-install-action@v0.11.0 + with: + enable-cache: "true" + skip-nix-installation: ${{ inputs.provider == 'Nutanix' }} + + - name: Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Set IP range based on provider + run: | + if [[ "${{ inputs.provider }}" == "Nutanix" ]]; then + CONTROL_PLANE_ENDPOINT_RANGE_START="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" + CONTROL_PLANE_ENDPOINT_RANGE_END="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" + fi + echo "CONTROL_PLANE_ENDPOINT_RANGE_START=$CONTROL_PLANE_ENDPOINT_RANGE_START" >> "$GITHUB_ENV" + echo "CONTROL_PLANE_ENDPOINT_RANGE_END=$CONTROL_PLANE_ENDPOINT_RANGE_END" >> "$GITHUB_ENV" + + - name: Get Control Plane endpoint IP + id: get-control-plane-endpoint-ip + run: | + control_plane_endpoint_ip=$(fping -g -u "$CONTROL_PLANE_ENDPOINT_RANGE_START" "$CONTROL_PLANE_ENDPOINT_RANGE_END" 2>/dev/null | shuf --head-count=1) + echo "control_plane_endpoint_ip=$control_plane_endpoint_ip" >> "$GITHUB_OUTPUT" + + - name: Run e2e tests + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }} + NUTANIX_USER: ${{ secrets.NUTANIX_USER }} + NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} + NUTANIX_PORT: ${{ vars.NUTANIX_PORT }} + NUTANIX_INSECURE: ${{ vars.NUTANIX_INSECURE }} + NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }} + NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }} + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }} + NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }} + CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} + + - if: success() || failure() # always run even if the previous step fails + name: Publish e2e test report + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'junit-e2e.xml' + check_name: 'e2e test report' + detailed_summary: true + require_passed_tests: true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d821469a1..ff68fc56e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1289,7 +1289,9 @@ func (in *NutanixNodeSpec) DeepCopy() *NutanixNodeSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto(out *NutanixPrismCentralEndpointCredentials) { +func (in *NutanixPrismCentralEndpointCredentials) DeepCopyInto( + out *NutanixPrismCentralEndpointCredentials, +) { *out = *in out.SecretRef = in.SecretRef } diff --git a/devbox.json b/devbox.json index 2106df97e..3bac79bcf 100644 --- a/devbox.json +++ b/devbox.json @@ -5,6 +5,7 @@ "coreutils@latest", "envsubst@latest", "findutils@latest", + "fping@latest", "gh@latest", "ginkgo@latest", "git@latest", diff --git a/devbox.lock b/devbox.lock index b975b2357..cc818dbc7 100644 --- a/devbox.lock +++ b/devbox.lock @@ -297,6 +297,54 @@ } } }, + "fping@latest": { + "last_modified": "2024-06-12T20:55:33Z", + "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping", + "source": "devbox-search", + "version": "5.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2" + } + } + }, "gh@latest": { "last_modified": "2024-05-30T12:09:21Z", "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh",