Skip to content

Commit

Permalink
Add another e2e workflow that gets run on self-hosted-github runner
Browse files Browse the repository at this point in the history
This is necessary as we want some e2e jobs to run on hosted runners
and some to run on self-hosted runners. The actions runner controller
does not support using multiple labels to target the runners
See: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
  • Loading branch information
thunderboltsid committed Jun 27, 2024
1 parent 2d2a3f7 commit 5c39242
Show file tree
Hide file tree
Showing 5 changed files with 196 additions and 2 deletions.
4 changes: 4 additions & 0 deletions .github/actionlint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
self-hosted-runner:
# Labels of self-hosted runner in array of string
labels:
- self-hosted-ncn-dind
40 changes: 38 additions & 2 deletions .github/workflows/checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ jobs:
- Docker
# Uncomment below once we have the ability to run e2e tests on other providers from GHA.
# - AWS
# - Nutanix
fail-fast: false
uses: ./.github/workflows/e2e.yml
with:
Expand All @@ -72,14 +71,32 @@ jobs:
contents: read
checks: write

# The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
# https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
e2e-quick-start-custom-runner:
strategy:
matrix:
provider:
- Nutanix
# Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
# - Vsphere (example)
fail-fast: false
uses: ./.github/workflows/e2e-custom-runner.yml
with:
provider: ${{ matrix.provider }}
focus: Quick start
secrets: inherit
permissions:
contents: read
checks: write

e2e-self-hosted:
strategy:
matrix:
provider:
- Docker
# Uncomment below once we have the ability to run e2e tests on other providers from GHA.
# - AWS
# - Nutanix
fail-fast: false
uses: ./.github/workflows/e2e.yml
with:
Expand All @@ -90,6 +107,25 @@ jobs:
contents: read
checks: write

# The actions runner controller does not support using multiple labels to target the runners so we have to create separate jobs for self-hosted runners.
# https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/using-actions-runner-controller-runners-in-a-workflow#using-runner-scale-set-names
e2e-self-hosted-custom-runner:
strategy:
matrix:
provider:
- Nutanix
# Uncomment below once we have the ability to run e2e tests on other providers via self-hosted runners.
# - Vsphere (example)
fail-fast: false
uses: ./.github/workflows/e2e-custom-runner.yml
with:
provider: ${{ matrix.provider }}
focus: Self-hosted
secrets: inherit
permissions:
contents: read
checks: write

lint-go:
runs-on: ubuntu-22.04
strategy:
Expand Down
105 changes: 105 additions & 0 deletions .github/workflows/e2e-custom-runner.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# Copyright 2024 Nutanix. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

on:
workflow_call:
inputs:
provider:
description: Infrastructure provider to run e2e tests with
type: string
required: true
skip:
description: e2e tests to skip
type: string
focus:
description: e2e tests to focus
type: string

jobs:
e2e-test:
runs-on: "self-hosted-ncn-dind"
permissions:
contents: read
checks: write
steps:
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Install Devbox
uses: jetify-com/devbox-install-action@v0.11.0
with:
enable-cache: true

- name: Go cache
uses: actions/cache@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Set IP range based on provider
run: |
if [[ "${{ inputs.provider }}" == "Nutanix" ]]; then
CONTROL_PLANE_ENDPOINT_RANGE_START="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_START }}"
CONTROL_PLANE_ENDPOINT_RANGE_END="${{ secrets.NUTANIX_CONTROL_PLANE_ENDPOINT_RANGE_END }}"
fi
echo "CONTROL_PLANE_ENDPOINT_RANGE_START=$CONTROL_PLANE_ENDPOINT_RANGE_START" >> $GITHUB_ENV
echo "CONTROL_PLANE_ENDPOINT_RANGE_END=$CONTROL_PLANE_ENDPOINT_RANGE_END" >> $GITHUB_ENV
- name: Get Control Plane endpoint IP
id: get-control-plane-endpoint-ip
run: |
# Get all available IPs in the range
available_ips=$(fping -g -u $CONTROL_PLANE_ENDPOINT_RANGE_START $CONTROL_PLANE_ENDPOINT_RANGE_END 2>/dev/null)
# Convert the list of available IPs to an array
available_ips_array=(${(f)ip_list})
# Get the number of available IPs
num_ips=${#available_ips_array[@]}
if [ $num_ips -eq 0 ]; then
echo "No available IPs found"
exit 1
fi
# Get a random index
random_index=$((RANDOM % num_ips +1))
# Get the random IP
control_plane_endpoint_ip=${available_ips_array[$random_index]}
# Set the random IP as an output variable
echo "control_plane_endpoint_ip=$control_plane_endpoint_ip" >> $GITHUB_OUTPUT
- name: Run e2e tests
run: devbox run -- make e2e-test E2E_LABEL='provider:${{ inputs.provider }}' E2E_SKIP='${{ inputs.skip }}' E2E_FOCUS='${{ inputs.focus }}'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
NUTANIX_ENDPOINT: ${{ secrets.NUTANIX_ENDPOINT }}
NUTANIX_USER: ${{ secrets.NUTANIX_USER }}
NUTANIX_PASSWORD: ${{ secrets.NUTANIX_PASSWORD }}
NUTANIX_PORT: ${{ variables.NUTANIX_PORT }}
NUTANIX_INSECURE: ${{ variables.NUTANIX_INSECURE }}
NUTANIX_PRISM_ELEMENT_CLUSTER_NAME: ${{ variables.NUTANIX_PRISM_ELEMENT_CLUSTER_NAME }}
NUTANIX_SUBNET_NAME: ${{ variables.NUTANIX_SUBNET_NAME }}
NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME: ${{ variables.NUTANIX_MACHINE_TEMPLATE_IMAGE_NAME }}
NUTANIX_STORAGE_CONTAINER_NAME: ${{ variables.NUTANIX_STORAGE_CONTAINER_NAME }}
CONTROL_PLANE_ENDPOINT_IP: ${{ steps.get-control-plane-endpoint-ip.outputs.control_plane_endpoint_ip }}

- if: success() || failure() # always run even if the previous step fails
name: Publish e2e test report
uses: mikepenz/action-junit-report@v4
with:
report_paths: 'junit-e2e.xml'
check_name: 'e2e test report'
detailed_summary: true
require_passed_tests: true
1 change: 1 addition & 0 deletions devbox.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"coreutils@latest",
"envsubst@latest",
"findutils@latest",
"fping@latest",
"gh@latest",
"ginkgo@latest",
"git@latest",
Expand Down
48 changes: 48 additions & 0 deletions devbox.lock
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,54 @@
}
}
},
"fping@latest": {
"last_modified": "2024-06-12T20:55:33Z",
"resolved": "github:NixOS/nixpkgs/a9858885e197f984d92d7fe64e9fff6b2e488d40#fping",
"source": "devbox-search",
"version": "5.2",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2",
"default": true
}
],
"store_path": "/nix/store/clxfp6jl0d2fs1bp2d1278534n2gixbj-fping-5.2"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2",
"default": true
}
],
"store_path": "/nix/store/ilzq042wih0h5vdzxcpf6sd826h37g6w-fping-5.2"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2",
"default": true
}
],
"store_path": "/nix/store/hrh3202f2njx3skj3xn33fish5az5691-fping-5.2"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2",
"default": true
}
],
"store_path": "/nix/store/2nr99jpa9g7b5z8pwj85awzh4qbhas28-fping-5.2"
}
}
},
"gh@latest": {
"last_modified": "2024-05-30T12:09:21Z",
"resolved": "github:NixOS/nixpkgs/aa61b27554a5fc282758bf0324781e3464ef2cde#gh",
Expand Down

0 comments on commit 5c39242

Please sign in to comment.