diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..ba68bf566 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - self-hosted-ncn-dind diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80e5471b9..2d6d9a642 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,6 @@ jobs: - Docker # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: @@ -72,6 +71,25 @@ jobs: contents: read checks: write + # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners. + # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names + e2e-quick-start-custom-runner: + strategy: + matrix: + provider: + - Nutanix + # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners. + # - Vsphere (example) + fail-fast: false + uses: ./.github/workflows/e2e-custom-runner.yml + with: + provider: ${{ matrix.provider }} + focus: Quick start + secrets: inherit + permissions: + contents: read + checks: write + e2e-self-hosted: strategy: matrix: @@ -79,7 +97,6 @@ jobs: - Docker # Uncomment below once we have the ability to run e2e tests on other providers from GHA. # - AWS - # - Nutanix fail-fast: false uses: ./.github/workflows/e2e.yml with: @@ -90,6 +107,25 @@ jobs: contents: read checks: write + # The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners. + # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names + e2e-self-hosted-custom-runner: + strategy: + matrix: + provider: + - Nutanix + # Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners. + # - Vsphere (example) + fail-fast: false + uses: ./.github/workflows/e2e-custom-runner.yml + with: + provider: ${{ matrix.provider }} + focus: Self-hosted + secrets: inherit + permissions: + contents: read + checks: write + lint-go: runs-on: ubuntu-22.04 strategy: diff --git a/.github/workflows/e2e-custom-runner.yml b/.github/workflows/e2e-custom-runner.yml new file mode 100644 index 000000000..4dd5357ff --- /dev/null +++ b/.github/workflows/e2e-custom-runner.yml @@ -0,0 +1,105 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +on: + workflow_call: + inputs: + provider: + description: Infrastructure provider to run e2e tests with + type: string + required: true + skip: + description: e2e tests to skip + type: string + focus: + description: e2e tests to focus + type: string + +jobs: + e2e-test: + runs-on: "self-hosted-ncn-dind" + permissions: + contents: read + checks: write + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Devbox + uses: jetify-com/devbox-install-action@v0.11.0 + with: + enable-cache: true + + - name: Go cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Set IP range based on provider + run: | + if [[ "${{ inputs.provider }}" == "Nutanix" ]]; then + CONTROL_PLANE_ENDPOINT_RANGE_START="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}" + CONTROL_PLANE_ENDPOINT_RANGE_END="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}" + fi + + echo "CONTROL_PLANE_ENDPOINT_RANGE_START=$CONTROL_PLANE_ENDPOINT_RANGE_START" >> $GITHUB_ENV + echo "CONTROL_PLANE_ENDPOINT_RANGE_END=$CONTROL_PLANE_ENDPOINT_RANGE_END" >> $GITHUB_ENV + + - name: Get Control Plane endpoint IP + id: get-control-plane-endpoint-ip + run: | + # Get all available IPs in the range + available_ips=$(fping -g -u $CONTROL_PLANE_ENDPOINT_RANGE_START $CONTROL_PLANE_ENDPOINT_RANGE_END 2>/dev/null) + + # Convert the list of available IPs to an array + available_ips_array=(${(f)ip_list}) + + # Get the number of available IPs + num_ips=${#available_ips_array[@]} + + if [ $num_ips -eq 0 ]; then + echo "No available IPs found" + exit 1 + fi + + # Get a random index + random_index=$((RANDOM % num_ips +1)) + + # Get the random IP + control_plane_endpoint_ip=${available_ips_array[$random_index]} + + # Set the random IP as an output variable + echo "control_plane_endpoint_ip=$control_plane_endpoint_ip" >> $GITHUB_OUTPUT + + - name: Run e2e tests + run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }} + NUTANIX_USER: ${{ secrets.NUTANIX_USER }} + NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }} + NUTANIX_PORT: ${{ variables.NUTANIX_PORT }} + NUTANIX_INSECURE: ${{ variables.NUTANIX_INSECURE }} + NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ variables.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }} + NUTANIX_SUBNET_NAME: ${{ variables.NUTANIX_SUBNET_NAME }} + NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ variables.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }} + NUTANIX_STORAGE_CONTAINER_NAME: ${{ variables.NUTANIX_STORAGE_CONTAINER_NAME }} + CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }} + + - if: success() || failure() # always run even if the previous step fails + name: Publish e2e test report + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'junit-e2e.xml' + check_name: 'e2e test report' + detailed_summary: true + require_passed_tests: true diff --git a/devbox.json b/devbox.json index 2106df97e..3bac79bcf 100644 --- a/devbox.json +++ b/devbox.json @@ -5,6 +5,7 @@ "coreutils@latest", "envsubst@latest", "findutils@latest", + "fping@latest", "gh@latest", "ginkgo@latest", "git@latest", diff --git a/devbox.lock b/devbox.lock index b975b2357..cc818dbc7 100644 --- a/devbox.lock +++ b/devbox.lock @@ -297,6 +297,54 @@ } } }, + "fping@latest": { + "last_modified": "2024-06-12T20:55:33Z", + "resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping", + "source": "devbox-search", + "version": "5.2", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2", + "default": true + } + ], + "store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2" + } + } + }, "gh@latest": { "last_modified": "2024-05-30T12:09:21Z", "resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh",