Skip to content

Commit

Permalink
Test the Kubernetes deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
stagnation authored and moroten committed Jan 30, 2024
1 parent 4e536b5 commit 98b5455
Show file tree
Hide file tree
Showing 9 changed files with 93 additions and 5 deletions.
8 changes: 8 additions & 0 deletions .github/workflows/master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
"name": "Installing Bazelisk",
"run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n mkdir -p ~/.cache/bazelisk &&\n curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
},
{
"name": "Install k3d",
"run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
},
{
"name": "Check out source code",
"uses": "actions/checkout@v1"
Expand Down Expand Up @@ -59,6 +63,10 @@
"name": "Test docker-compose deployment",
"run": "tools/test-deployment-docker-compose.sh"
},
{
"name": "Test Kubernetes deployment",
"run": "tools/test-deployment-kubernetes"
},
{
"name": "linux_amd64: build and test",
"run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/pull-requests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
"name": "Installing Bazelisk",
"run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n mkdir -p ~/.cache/bazelisk &&\n curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
},
{
"name": "Install k3d",
"run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
},
{
"name": "Check out source code",
"uses": "actions/checkout@v1"
Expand Down Expand Up @@ -59,6 +63,10 @@
"name": "Test docker-compose deployment",
"run": "tools/test-deployment-docker-compose.sh"
},
{
"name": "Test Kubernetes deployment",
"run": "tools/test-deployment-kubernetes"
},
{
"name": "linux_amd64: build and test",
"run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."
Expand Down
2 changes: 1 addition & 1 deletion tools/diff-docker-and-k8s-configs.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bash

set -eu -o pipefail
set -eu -o pipefail -E

diff_config() {
k8s_config="$1"; shift
Expand Down
8 changes: 8 additions & 0 deletions tools/github_workflows/workflows_template.libsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@
echo "~/.cache/bazelisk" >> ${GITHUB_PATH}
|||,
},
{
name: 'Install k3d',
run: 'curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash',
},
{
name: 'Check out source code',
uses: 'actions/checkout@v1',
Expand Down Expand Up @@ -119,6 +123,10 @@
name: 'Test docker-compose deployment',
run: 'tools/test-deployment-docker-compose.sh',
},
{
name: 'Test Kubernetes deployment',
run: 'tools/test-deployment-kubernetes',
},
] + std.flattenArrays([
[{
name: platform.name + ': build and test',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
set -eEuxo pipefail

set -eux -o pipefail -E

bazel_version="$1"

Expand Down
2 changes: 1 addition & 1 deletion tools/remote-toolchains/extract-bazel-auto-toolchains.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# `.bazelversion` in the repository root.
#
# Example usage: ./extract-bazel-auto-toolchains.sh ubuntu-act-22-04 ghcr.io/catthehacker/ubuntu:act-22.04
set -eu -o pipefail
set -eu -o pipefail -E

script_dir="$(dirname "${BASH_SOURCE[0]}")"

Expand Down
2 changes: 1 addition & 1 deletion tools/test-deployment-docker-compose.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash

# Verifies that the docker-compose deployment works.
set -eux -o pipefail
set -eux -o pipefail -E

script_dir=$(dirname "${BASH_SOURCE[0]}")
cd "${script_dir}/../docker-compose"
Expand Down
61 changes: 61 additions & 0 deletions tools/test-deployment-kubernetes
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# Verifies that the Kuberenetes deployment works.

set -eux -o pipefail -E

script_dir=$(dirname "${BASH_SOURCE[0]}")
root="$(realpath "$script_dir"/..)"
cd "$root"

# --- Launch Kubernetes deployment with k3d ---
kubeconfig="$(realpath kubernetes/kubeconfig.yaml)"
if test -e "$kubeconfig"; then
echo >&2 "ERROR: $kubeconfig already exists"
exit 1
fi
export KUBECONFIG="$kubeconfig"

k3d cluster create bb-deployments-test

cleanup() {
EXIT_STATUS=$?
# Ignore errors when tearing down.
set +e
k3d cluster delete bb-deployments-test
rm "$kubeconfig"
return "$EXIT_STATUS"
}
trap cleanup EXIT

# Deploy Buildbarn.
kubectl apply -k kubernetes
kubectl rollout status --watch --namespace=buildbarn --timeout=300s statefulset
kubectl rollout status --watch --namespace=buildbarn --timeout=300s deployment

# Find the frontend.
bb_frontend_ip=$(kubectl get service --namespace=buildbarn frontend -o json | jq --raw-output '.status.loadBalancer.ingress[0].ip')
bb_frontend_address="grpc://$bb_frontend_ip:8980"
ingress_ip=$(kubectl get service --namespace=kube-system traefik -o json | jq --raw-output '.status.loadBalancer.ingress[0].ip')

# --- Test that the bb-browser and bb-scheduler UIs are responsive. ---
curl --fail --header 'Host: bb-browser.example.com' "http://$ingress_ip/" | grep '<title>Buildbarn Browser</title>'
curl --fail --header 'Host: bb-scheduler.example.com' "http://$ingress_ip/" | grep '<title>Buildbarn Scheduler</title>'

# --- Run remote execution ---
bazel_command_log="$(bazel info output_base)/command.log"
bazel clean
bazel test --color=no --curses=no --config=remote-ubuntu-22-04 --disk_cache= --remote_executor="$bb_frontend_address" @abseil-hello//:hello_test
# Make sure there are remote executions but no cache hits.
# INFO: 39 processes: 9 internal, 30 remote.
grep -E '^INFO: [0-9]+ processes: .*[0-9]+ remote[.,]' \
"$bazel_command_log" \
| grep -v 'remote cache hit'

# --- Check that the hardlinking workers are available ---
bazel clean
bazel test --color=no --curses=no --config=remote-ubuntu-22-04 --remote_instance_name=hardlinking --disk_cache= --remote_executor="$bb_frontend_address" @abseil-hello//:hello_test
# Make sure there are remote executions but no cache hits.
# INFO: 39 processes: 9 internal, 30 remote.
grep -E '^INFO: [0-9]+ processes: .*[0-9]+ remote[.,]' \
"$bazel_command_log" \
| grep -v 'remote cache hit'
4 changes: 3 additions & 1 deletion tools/update-container-image-versions.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/usr/bin/env bash
set -eu -o pipefail

set -eu -o pipefail -E

# Updates the image version in the Docker Compose and Kubernetes deployments.
#
# Run this script as soon as go.mod has been updated.
Expand Down

0 comments on commit 98b5455

Please sign in to comment.