From e6d23e34c446e61f755c0e83e8cb402b1d732c68 Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Wed, 26 Jun 2024 16:01:58 +0200 Subject: [PATCH] Update e2e workflow to conditionally run on self-hosted-github runner This is necessary as we want some e2e jobs to run on hosted runners and some to run on self-hosted runners. The actions runner controller does not support using multiple labels to target the runners See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names ci: Run Nutanix provider e2e tests on self-hosted runner --- .github/actionlint.yaml | 7 ++ .github/workflows/checks.yml | 6 +- .github/workflows/e2e.yml | 66 +++++++++--- devbox.json | 1 + devbox.lock | 48 +++++++++ test/e2e/self_hosted_test.go | 192 +++++++++++++++++------------------ 6 files changed, 208 insertions(+), 112 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..8c04ab7e5 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,7 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - self-hosted-ncn-dind diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 40844fd4b..4ed3bcce9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -59,14 +59,15 @@ jobs: matrix: provider: - Docker + - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} focus: Quick start + runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit permissions: contents: read @@ -77,14 +78,15 @@ jobs: matrix: provider: - Docker + - Nutanix # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: provider: ${{ matrix.provider }} focus: Self-hosted + runs-on: ${{ matrix.provider == 'Nutanix' && 'self-hosted-ncn-dind' || 'ubuntu-22.04' }} secrets: inherit permissions: contents: read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 487e9af92..218624029 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -14,10 +14,14 @@ on: focus: description: e2e tests to focus type: string + runs-on: + description: The runner to run the e2e tests on + type: string + required: true jobs: e2e-test: - runs-on: ubuntu-22.04 + runs-on: ${{ inputs.runs-on }} permissions: contents: read checks: write @@ -27,41 +31,65 @@ jobs: with: fetch-depth: 0 - - name: Install devbox - uses: jetify-com/devbox-install-action@v0.11.0 + - uses: cachix/install-nix-action@V27 + if: inputs.provider == 'Nutanix' with: - enable-cache: true + github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Go cache - uses: actions/cache@v4 + - name: Install devbox + uses: jetify-com/devbox-install-action@v0.11.0 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + enable-cache: "true" + skip-nix-installation: ${{ inputs.provider == 'Nutanix' }} # The default disk size of these runners is ~14GB, this is not enough to run the e2e tests. # Cleanup the disk, see upstream discussion https://github.com/actions/runner-images/issues/2840. - name: Cleanup Disk Space + if: inputs.runs-on != 'self-hosted-ncn-dind' run: | echo "Before removing files:" df -h sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf "${AGENT_TOOLSDIRECTORY}" echo "After removing files:" df -h + - name: Get Control Plane endpoint IP + id: get-control-plane-endpoint-ip + if: inputs.provider == 'Nutanix' + run: | + CONTROL_PLANE_ENDPOINT_RANGE_START="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" + CONTROL_PLANE_ENDPOINT_RANGE_END="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" + control_plane_endpoint_ip="$(devbox run -- fping -g -u "${CONTROL_PLANE_ENDPOINT_RANGE_START}" "${CONTROL_PLANE_ENDPOINT_RANGE_END}" 2>/dev/null | devbox run -- shuf --head-count=1)" + echo "control_plane_endpoint_ip=${control_plane_endpoint_ip}" >> "${GITHUB_OUTPUT}" + - name: Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run e2e tests run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }} - NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} NUTANIX_USER: ${{ secrets.NUTANIX_USER }} + NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} + NUTANIX_PORT: ${{ vars.NUTANIX_PORT }} + NUTANIX_INSECURE: ${{ vars.NUTANIX_INSECURE }} + NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ vars.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }} + NUTANIX_SUBNET_NAME: ${{ vars.NUTANIX_SUBNET_NAME }} + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ vars.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }} + NUTANIX_STORAGE_CONTAINER_NAME: ${{ vars.NUTANIX_STORAGE_CONTAINER_NAME }} + CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} - if: success() || failure() # always run even if the previous step fails name: Publish e2e test report @@ -71,3 +99,13 @@ jobs: check_name: 'e2e test report' detailed_summary: true require_passed_tests: true + + - if: success() || failure() # always run even if the previous step fails + name: Upload e2e test artifacts + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + overwrite: false + name: ${{ inputs.provider }} ${{ inputs.focus }} e2e artifacts + path: | + _artifacts diff --git a/devbox.json b/devbox.json index 369f18c24..0a80a8fc3 100644 --- a/devbox.json +++ b/devbox.json @@ -6,6 +6,7 @@ "crane@latest", "envsubst@latest", "findutils@latest", + "fping@latest", "gh@latest", "ginkgo@latest", "git@latest", diff --git a/devbox.lock b/devbox.lock index f5e4ccb95..8e54bb4c7 100644 --- a/devbox.lock +++ b/devbox.lock @@ -381,6 +381,54 @@ } } }, + "fping@latest": { + "last_modified": "2024-06-12T20:55:33Z", + "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping", + "source": "devbox-search", + "version": "5.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2" + } + } + }, "gh@latest": { "last_modified": "2024-05-30T12:09:21Z", "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh", diff --git a/test/e2e/self_hosted_test.go b/test/e2e/self_hosted_test.go index 8e15d69a9..9b465a89d 100644 --- a/test/e2e/self_hosted_test.go +++ b/test/e2e/self_hosted_test.go @@ -22,103 +22,103 @@ import ( ) var _ = Describe("Self-hosted", Serial, func() { - provider := "Docker" - lowercaseProvider := strings.ToLower(provider) - for _, cniProvider := range []string{"Cilium", "Calico"} { - for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { - strategy := "" - switch addonStrategy { - case "HelmAddon": - strategy = "helm-addon" - case "ClusterResourceSet": - strategy = "crs" - default: - Fail("unknown addon strategy: " + addonStrategy) - } - flavour := fmt.Sprintf( - "topology-%s-%s", - strings.ToLower(cniProvider), - strategy, - ) - Context( - flavour, - Label("provider:"+provider), - Label("cni:"+cniProvider), - Label("addonStrategy:"+addonStrategy), - func() { - framework.SelfHostedSpec(ctx, func() framework.SelfHostedSpecInput { - return framework.SelfHostedSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - Flavor: flavour, - InfrastructureProvider: ptr.To(lowercaseProvider), - PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { - By( - "Waiting for all requested addons to be ready in workload cluster", - ) - workloadCluster := capiframework.GetClusterByName( - ctx, - capiframework.GetClusterByNameInput{ - Namespace: cluster.GetNamespace(), - Name: cluster.GetName(), - Getter: proxy.GetClient(), - }, - ) - Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) - clusterVars := variables.ClusterVariablesToVariablesMap( - workloadCluster.Spec.Topology.Variables, - ) - addonsConfig, err := variables.Get[apivariables.Addons]( - clusterVars, - v1alpha1.ClusterConfigVariableName, - "addons", - ) - Expect(err).ToNot(HaveOccurred()) - WaitForAddonsToBeReadyInWorkloadCluster( - ctx, - WaitForAddonsToBeReadyInWorkloadClusterInput{ - AddonsConfig: addonsConfig, - ClusterProxy: proxy, - WorkloadCluster: workloadCluster, - InfrastructureProvider: lowercaseProvider, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), - DaemonSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-daemonset", - ), - HelmReleaseIntervals: e2eConfig.GetIntervals( - flavour, - "wait-helmrelease", - ), - ClusterResourceSetIntervals: e2eConfig.GetIntervals( - flavour, - "wait-clusterresourceset", - ), - }, - ) + for _, provider := range []string{"Docker", "Nutanix"} { + lowercaseProvider := strings.ToLower(provider) + for _, cniProvider := range []string{"Cilium", "Calico"} { + for _, addonStrategy := range []string{"HelmAddon", "ClusterResourceSet"} { + strategy := "" + switch addonStrategy { + case "HelmAddon": + strategy = "helm-addon" + case "ClusterResourceSet": + strategy = "crs" + default: + Fail("unknown addon strategy: " + addonStrategy) + } + flavour := fmt.Sprintf( + "topology-%s-%s", + strings.ToLower(cniProvider), + strategy, + ) + Context( + flavour, + Label("provider:"+provider), + Label("cni:"+cniProvider), + Label("addonStrategy:"+addonStrategy), + func() { + framework.SelfHostedSpec(ctx, func() framework.SelfHostedSpecInput { + return framework.SelfHostedSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: flavour, + InfrastructureProvider: ptr.To(lowercaseProvider), + PostClusterMoved: func(proxy capiframework.ClusterProxy, cluster *clusterv1.Cluster) { + By( + "Waiting for all requested addons to be ready in workload cluster", + ) + workloadCluster := capiframework.GetClusterByName( + ctx, + capiframework.GetClusterByNameInput{ + Namespace: cluster.GetNamespace(), + Name: cluster.GetName(), + Getter: proxy.GetClient(), + }, + ) + Expect(workloadCluster.Spec.Topology).ToNot(BeNil()) + clusterVars := variables.ClusterVariablesToVariablesMap( + workloadCluster.Spec.Topology.Variables, + ) + addonsConfig, err := variables.Get[apivariables.Addons]( + clusterVars, + v1alpha1.ClusterConfigVariableName, + "addons", + ) + Expect(err).ToNot(HaveOccurred()) + WaitForAddonsToBeReadyInWorkloadCluster( + ctx, + WaitForAddonsToBeReadyInWorkloadClusterInput{ + AddonsConfig: addonsConfig, + ClusterProxy: proxy, + WorkloadCluster: workloadCluster, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + DaemonSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-daemonset", + ), + HelmReleaseIntervals: e2eConfig.GetIntervals( + flavour, + "wait-helmrelease", + ), + ClusterResourceSetIntervals: e2eConfig.GetIntervals( + flavour, + "wait-clusterresourceset", + ), + }, + ) - WaitForCoreDNSToBeReadyInWorkloadCluster( - ctx, - WaitForCoreDNSToBeReadyInWorkloadClusterInput{ - WorkloadCluster: workloadCluster, - ClusterProxy: proxy, - DeploymentIntervals: e2eConfig.GetIntervals( - flavour, - "wait-deployment", - ), - }, - ) - }, - } - }) - }, - ) + WaitForCoreDNSToBeReadyInWorkloadCluster( + ctx, + WaitForCoreDNSToBeReadyInWorkloadClusterInput{ + WorkloadCluster: workloadCluster, + ClusterProxy: proxy, + DeploymentIntervals: e2eConfig.GetIntervals( + flavour, + "wait-deployment", + ), + }, + ) + }, + } + }) + }, + ) + } } } })