diff --git a/.github/actions/vagrant-setup/action.yaml b/.github/actions/vagrant-setup/action.yaml index 28a8f2b0bb5f..37f268809e20 100644 --- a/.github/actions/vagrant-setup/action.yaml +++ b/.github/actions/vagrant-setup/action.yaml @@ -20,6 +20,14 @@ runs: run: | sudo apt-get build-dep -y vagrant ruby-libvirt sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev + # This is a workaround for the libvirt group not being available in the current shell + # https://github.com/actions/runner-images/issues/7670#issuecomment-1900711711 + - name: Make the libvirt socket rw accessible to everyone + shell: bash + run: | + sudo chmod a+rw /var/run/libvirt/libvirt-sock + + - name: Install vagrant-libvirt plugin shell: bash - run: sudo vagrant plugin install vagrant-libvirt \ No newline at end of file + run: vagrant plugin install vagrant-libvirt \ No newline at end of file diff --git a/.github/workflows/build-k3s.yaml b/.github/workflows/build-k3s.yaml index 19c40b5ee897..aa702fd454ec 100644 --- a/.github/workflows/build-k3s.yaml +++ b/.github/workflows/build-k3s.yaml @@ -7,6 +7,10 @@ on: type: boolean required: false default: false + upload-image: + type: boolean + required: false + default: false permissions: contents: read @@ -18,11 +22,14 @@ jobs: timeout-minutes: 20 steps: - name: Checkout K3s - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build K3s binary run: | DOCKER_BUILDKIT=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 make sha256sum dist/artifacts/k3s | sed 's|dist/artifacts/||' > dist/artifacts/k3s.sha256sum + - name: Build K3s image + if: inputs.upload-image == true + run: make package-image - name: bundle repo if: inputs.upload-repo == true run: | @@ -30,13 +37,16 @@ jobs: mv ../k3s-repo.tar.gz . - name: "Upload K3s directory" if: inputs.upload-repo == true - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: k3s-repo.tar.gz path: k3s-repo.tar.gz + - name: "Save K3s image" + if: inputs.upload-image == true + run: docker image save rancher/k3s -o ./dist/artifacts/k3s-image.tar - name: "Upload K3s binary" if: inputs.upload-repo == false - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: k3s path: dist/artifacts/k3s* \ No newline at end of file diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000000..9c4767c9df57 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,115 @@ +name: E2E Test Coverage +on: + push: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/e2e**" + - ".github/**" + - "!.github/workflows/e2e.yaml" + pull_request: + paths-ignore: + - "**.md" + - "channel.yaml" + - "install.sh" + - "tests/**" + - "!tests/e2e**" + - ".github/**" + - "!.github/workflows/e2e.yaml" + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + build: + uses: ./.github/workflows/build-k3s.yaml + with: + upload-image: true + e2e: + name: "E2E Tests" + needs: build + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + # TODO fix embeddedmirror and add it to the matrix + etest: [startup, s3, btrfs, externalip, privateregistry] + max-parallel: 3 + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: {fetch-depth: 1} + + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup + - name: "Vagrant Cache" + uses: actions/cache@v4 + with: + path: | + ~/.vagrant.d/boxes + key: vagrant-box-ubuntu-2204 + - name: "Vagrant Plugin(s)" + run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: false + - name: "Download k3s binary" + uses: actions/download-artifact@v4 + with: + name: k3s + path: ./dist/artifacts + + - name: Run ${{ matrix.etest }} Test + env: + E2E_GOCOVER: "true" + run: | + chmod +x ./dist/artifacts/k3s + cd tests/e2e/${{ matrix.etest }} + go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local + - name: On Failure, Launch Debug Session + uses: lhotari/action-upterm@v1 + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 + - name: Upload Results To Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: tests/e2e/${{ matrix.etest }}/coverage.out + flags: e2etests # optional + verbose: true # optional (default = false) + docker: + needs: build + name: Docker Tests + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + dtest: [basics, bootstraptoken, cacerts, lazypull, upgrade] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: "Download k3s image" + uses: actions/download-artifact@v4 + with: + name: k3s + path: ./dist/artifacts + - name: Load k3s image + run: docker image load -i ./dist/artifacts/k3s-image.tar + - name: Run ${{ matrix.dtest }} Test + run: | + chmod +x ./dist/artifacts/k3s + . ./tests/docker/test-helpers + . ./tests/docker/test-run-${{ matrix.dtest }} + echo "Did test-run-${{ matrix.dtest }} pass $?" \ No newline at end of file diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index 5a534f1b2cd9..3353bae5e823 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -39,72 +39,61 @@ jobs: INSTALL_K3S_SKIP_DOWNLOAD: binary steps: - name: "Checkout" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: {fetch-depth: 1} - name: Set up vagrant and libvirt uses: ./.github/actions/vagrant-setup - - name: "Vagrant Cache" uses: actions/cache@v4 with: path: | - /tmp/boxes + ~/.vagrant.d/boxes key: vagrant-box-${{ matrix.vm }} - # Workaround for https://github.com/actions/cache/issues/1319 - - name: Move vagrant cache to /root - run: | - mkdir -p /tmp/boxes - sudo rm -rf /root/.vagrant.d/boxes - sudo mv -f /tmp/boxes /root/.vagrant.d - name: "Vagrant Plugin(s)" - run: sudo vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp + run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp - name: "Download k3s binary" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: k3s path: tests/install/${{ matrix.vm }} - name: "Vagrant Up" - run: sudo vagrant up --no-provision + run: vagrant up --no-provision - name: "Upload k3s binary to VM" run: | chmod +x k3s - sudo vagrant scp k3s /tmp/k3s - sudo vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" - sudo vagrant provision --provision-with=k3s-upload + vagrant scp k3s /tmp/k3s + vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" + vagrant provision --provision-with=k3s-upload - name: Add binary to PATH if: matrix.vm == 'centos-7' || matrix.vm == 'rocky-8' || matrix.vm == 'opensuse-leap' - run: sudo vagrant provision --provision-with=add-bin-path + run: vagrant provision --provision-with=add-bin-path - name: "⏩ Install K3s" run: | - sudo vagrant provision --provision-with=k3s-prepare - sudo vagrant provision --provision-with=k3s-install + vagrant provision --provision-with=k3s-prepare + vagrant provision --provision-with=k3s-install if [ ${{ matrix.vm }} = 'opensuse-microos' ]; then vagrant reload --no-provision; fi - name: "⏳ Node" - run: sudo vagrant provision --provision-with=k3s-wait-for-node + run: vagrant provision --provision-with=k3s-wait-for-node - name: "⏳ CoreDNS" - run: sudo vagrant provision --provision-with=k3s-wait-for-coredns + run: vagrant provision --provision-with=k3s-wait-for-coredns - name: "⏳ Local Storage" - run: sudo vagrant provision --provision-with=k3s-wait-for-local-storage + run: vagrant provision --provision-with=k3s-wait-for-local-storage continue-on-error: true - name: "⏳ Metrics Server" - run: sudo vagrant provision --provision-with=k3s-wait-for-metrics-server + run: vagrant provision --provision-with=k3s-wait-for-metrics-server continue-on-error: true - name: "⏳ Traefik" - run: sudo vagrant provision --provision-with=k3s-wait-for-traefik + run: vagrant provision --provision-with=k3s-wait-for-traefik continue-on-error: true - name: "k3s-status" - run: sudo vagrant provision --provision-with=k3s-status + run: vagrant provision --provision-with=k3s-status - name: "k3s-procps" - run: sudo vagrant provision --provision-with=k3s-procps + run: vagrant provision --provision-with=k3s-procps - name: Cleanup VM - run: sudo vagrant destroy -f + run: vagrant destroy -f - name: On Failure, launch debug session uses: lhotari/action-upterm@v1 if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. - wait-timeout-minutes: 5 - - name: Copy out vagrant boxes for cache - run: | - sudo mv -f /root/.vagrant.d/boxes /tmp/boxes - sudo chmod -R 777 /tmp/boxes \ No newline at end of file + wait-timeout-minutes: 5 \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index a9e3f378d3f9..71cc2b355df3 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -27,10 +27,10 @@ permissions: jobs: build: uses: ./.github/workflows/build-k3s.yaml - test: + itest: needs: build name: Integration Tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 45 strategy: fail-fast: false @@ -39,13 +39,13 @@ jobs: max-parallel: 3 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Install Go uses: ./.github/actions/setup-go - name: "Download k3s binary" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: k3s path: ./dist/artifacts @@ -58,4 +58,4 @@ jobs: if: ${{ failure() }} with: ## If no one connects after 5 minutes, shut down server. - wait-timeout-minutes: 5 \ No newline at end of file + wait-timeout-minutes: 5 diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index e6bf73da6324..c318fb0b3b0a 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -25,21 +25,15 @@ jobs: INSTALL_K3S_CHANNEL: ${{ matrix.channel }} steps: - name: "Checkout" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: {fetch-depth: 1} - name: Set up vagrant and libvirt uses: ./.github/actions/vagrant-setup - # Workaround for https://github.com/actions/cache/issues/1319 - - name: Move vagrant cache to /root - run: | - mkdir -p /tmp/boxes - sudo rm -rf /root/.vagrant.d/boxes - sudo mv -f /tmp/boxes /root/.vagrant.d - name: "Vagrant Cache" uses: actions/cache@v4 with: path: | - /tmp/boxes + ~/.vagrant.d/boxes key: vagrant-box-${{ matrix.vm }} id: vagrant-cache - name: "Vagrant Plugin(s)" @@ -62,8 +56,4 @@ jobs: - name: "k3s-status" run: vagrant provision --provision-with=k3s-status - name: "k3s-procps" - run: vagrant provision --provision-with=k3s-procps - - name: Copy out vagrant boxes for cache - run: | - sudo mv -f /root/.vagrant.d/boxes /tmp/boxes - sudo chmod -R 777 /tmp/boxes \ No newline at end of file + run: vagrant provision --provision-with=k3s-procps \ No newline at end of file diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml deleted file mode 100644 index 405da9b4284c..000000000000 --- a/.github/workflows/snapshotter.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: Snapshotter -on: - push: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/snapshotter/**" - - ".github/**" - - "!.github/workflows/snapshotter.yaml" - pull_request: - paths-ignore: - - "**.md" - - "channel.yaml" - - "install.sh" - - "tests/**" - - "!tests/snapshotter/**" - - ".github/**" - - "!.github/workflows/snapshotter.yaml" - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - build: - uses: ./.github/workflows/build-k3s.yaml - test: - name: "Smoke Test" - needs: build - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - matrix: - vm: [opensuse-leap] - snapshotter: [btrfs] - max-parallel: 1 - defaults: - run: - working-directory: tests/snapshotter/${{ matrix.snapshotter }}/${{ matrix.vm }} - env: - VAGRANT_EXPERIMENTAL: disks - steps: - - name: "Checkout" - uses: actions/checkout@v3 - with: { fetch-depth: 1 } - - name: "Download Binary" - uses: actions/download-artifact@v3 - with: { name: k3s, path: dist/artifacts/ } - - name: Set up vagrant and libvirt - uses: ./.github/actions/vagrant-setup - # Workaround for https://github.com/actions/cache/issues/1319 - - name: Move vagrant cache to /root - run: | - mkdir -p /tmp/boxes - sudo rm -rf /root/.vagrant.d/boxes - sudo mv -f /tmp/boxes /root/.vagrant.d - - name: "Vagrant Cache" - uses: actions/cache@v4 - with: - path: | - /tmp/boxes - /tmp/gems - key: vagrant-box-${{ matrix.vm }} - - name: "Vagrant Plugin(s)" - run: sudo vagrant plugin install vagrant-k3s - - name: "Vagrant Up ⏩ Install K3s" - run: sudo vagrant up - - name: "⏳ Node" - run: sudo vagrant provision --provision-with=k3s-wait-for-node - - name: "⏳ CoreDNS" - run: sudo vagrant provision --provision-with=k3s-wait-for-coredns - - name: "k3s-status" # kubectl get node,all -A -o wide - run: sudo vagrant provision --provision-with=k3s-status - - name: "k3s-snapshots" # if no snapshots then we fail - run: sudo vagrant provision --provision-with=k3s-snapshots - - name: Copy out vagrant boxes for cache - run: | - sudo mv -f /root/.vagrant.d/boxes /tmp/boxes - sudo chmod -R 777 /tmp/boxes \ No newline at end of file diff --git a/.github/workflows/unitcoverage.yaml b/.github/workflows/unitcoverage.yaml index 9e8d87a152b2..511ee752ee85 100644 --- a/.github/workflows/unitcoverage.yaml +++ b/.github/workflows/unitcoverage.yaml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 20 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Install Go diff --git a/.github/workflows/updatecli.yaml b/.github/workflows/updatecli.yaml index b350fa60f3be..032b660ff6cf 100644 --- a/.github/workflows/updatecli.yaml +++ b/.github/workflows/updatecli.yaml @@ -20,7 +20,7 @@ jobs: if: github.ref == 'refs/heads/master' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Go uses: actions/setup-go@v4 diff --git a/Dockerfile.test b/Dockerfile.test index 541665f65f2a..afc946995538 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -40,7 +40,7 @@ FROM vagrantlibvirt/vagrant-libvirt:0.12.1 AS test-e2e RUN apt-get update && apt-get install -y docker.io ENV VAGRANT_DISABLE_STRICT_DEPENDENCY_ENFORCEMENT=1 RUN vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp -RUN vagrant box add generic/ubuntu2004 --provider libvirt --force +RUN vagrant box add generic/ubuntu2204 --provider libvirt --force RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"; \ chmod +x ./kubectl; \ mv ./kubectl /usr/local/bin/kubectl diff --git a/scripts/test b/scripts/test index d90d843386cc..684e1398003e 100755 --- a/scripts/test +++ b/scripts/test @@ -3,7 +3,7 @@ set -e -x cd $(dirname $0)/.. . ./scripts/version.sh -. ./scripts/test-helpers +. ./tests/docker/test-helpers # sysctl commands sysctl -w fs.inotify.max_queued_events=16384 @@ -20,30 +20,33 @@ mkdir -p $artifacts docker ps # --- +# Only run basic tests on non amd64 archs, we use GitHub Actions for amd64 +if [ "$ARCH" != 'amd64' ]; then -. ./scripts/test-run-basics -echo "Did test-run-basics $?" + . ./tests/docker/test-run-basics + echo "Did test-run-basics $?" -. ./scripts/test-run-compat -echo "Did test-run-compat $?" + . ./tests/docker/test-run-cacerts + echo "Did test-run-cacerts $?" -. ./scripts/test-run-hardened -echo "Did test-run-hardened $?" + . ./tests/docker/test-run-bootstraptoken + echo "Did test-run-bootstraptoken $?" -. ./scripts/test-run-cacerts -echo "Did test-run-cacerts $?" + . ./tests/docker/test-run-upgrade + echo "Did test-run-upgrade $?" -. ./scripts/test-run-bootstraptoken -echo "Did test-run-bootstraptoken $?" + . ./tests/docker/test-run-lazypull + echo "Did test-run-lazypull $?" +fi -. ./scripts/test-run-upgrade -echo "Did test-run-upgrade $?" +. ./tests/docker/test-run-compat +echo "Did test-run-compat $?" -. ./scripts/test-run-etcd -echo "Did test-run-etcd $?" +. ./tests/docker/test-run-hardened +echo "Did test-run-hardened $?" -. ./scripts/test-run-lazypull -echo "Did test-run-lazypull $?" +. ./tests/docker/test-run-etcd +echo "Did test-run-etcd $?" # --- diff --git a/scripts/test-certs-openssl.cnf b/scripts/test-certs-openssl.cnf deleted file mode 100644 index 222288ec6da8..000000000000 --- a/scripts/test-certs-openssl.cnf +++ /dev/null @@ -1,72 +0,0 @@ -# db OpenSSL configuration file. -SAN = "IP:127.0.0.1, IP:172.17.0.1, DNS:host.docker.internal" -dir = . - -[ ca ] -default_ca = db_ca - -[ db_ca ] -certs = $dir/certs -certificate = $dir/certs/ca.crt -crl = $dir/crl.pem -crl_dir = $dir/crl -crlnumber = $dir/crlnumber -database = $dir/index.txt -email_in_dn = no -new_certs_dir = $dir/newcerts -private_key = $dir/private/ca.key -serial = $dir/serial -RANDFILE = $dir/private/.rand -name_opt = ca_default -cert_opt = ca_default -default_days = 3650 -default_crl_days = 30 -default_md = sha512 -preserve = no -policy = policy_db - -[ policy_db ] -organizationName = optional -commonName = supplied - -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes -x509_extensions = v3_ca -string_mask = utf8only -req_extensions = db_client - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_default = US -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (FQDN) -0.organizationName = Organization Name (eg, company) -0.organizationName_default = db-ca - -[ req_attributes ] - -[ v3_ca ] -basicConstraints = CA:true -keyUsage = keyCertSign,cRLSign -subjectKeyIdentifier = hash - -[ db_client ] -basicConstraints = CA:FALSE -extendedKeyUsage = clientAuth -keyUsage = digitalSignature, keyEncipherment - -[ db_peer ] -basicConstraints = CA:FALSE -extendedKeyUsage = clientAuth, serverAuth -keyUsage = digitalSignature, keyEncipherment -subjectAltName = ${ENV::SAN} - -[ db_server ] -basicConstraints = CA:FALSE -extendedKeyUsage = clientAuth, serverAuth -keyUsage = digitalSignature, keyEncipherment -subjectAltName = ${ENV::SAN} \ No newline at end of file diff --git a/scripts/test-unit b/scripts/test-unit deleted file mode 100755 index 429755243686..000000000000 --- a/scripts/test-unit +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -go test -v ./pkg/... -run Unit diff --git a/tests/TESTING.md b/tests/TESTING.md index 9b1fc8c24bb4..951f11467607 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -3,6 +3,7 @@ Testing in K3s comes in 5 forms: - [Unit](#unit-tests) - [Integration](#integration-tests) +- [Docker](#docker-tests) - [Smoke](#smoke-tests) - [Performance](#performance) - [End-to-End (E2E)](#end-to-end-e2e-tests) @@ -58,9 +59,15 @@ See [integration/README.md](./integration/README.md) for more info. ___ -## Smoke Tests +## Docker Tests -Smoke tests are a collection of tests defined under the [tests](./tests) and fall into two categories: install and snapshotter. These tests are used to validate the installation and operation of K3s on a variety of operating systems. The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path" scenarios. The test themses are Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers: +Docker tests run clusters of K3s nodes as containers and test basic functionality. These tests are run in the Drone CI pipeline `test` stage. + +___ + +## Install Tests + +Install tests are a collection of tests defined under the [tests/install](./tests/install). These tests are used to validate the installation and operation of K3s on a variety of operating systems. The test themselves are Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers: - [Install Script](install) :arrow_right: scheduled nightly and on an install script change - [CentOS 7](install/centos-7) (stand-in for RHEL 7) @@ -69,16 +76,11 @@ Smoke tests are a collection of tests defined under the [tests](./tests) and fal - [Fedora 37](install/fedora) - [Leap 15.5](install/opensuse-leap) (stand-in for SLES) - [Ubuntu 22.04](install/ubuntu-2204) -- [Snapshotter](snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change - - [BTRFS](snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs)) - - [Leap 15.4](../tests/snapshotter/btrfs/opensuse-leap) ## Format When adding new installer test(s) please copy the prevalent style for the `Vagrantfile`. Ideally, the boxes used for additional assertions will support the default `libvirt` provider which -enables them to be used by our Github Actions Workflow(s). See: -- [install.yaml](../.github/workflows/install.yaml). -- [snapshotter.yaml](../.github/workflows/snapshotter.yaml). +enables them to be used by our GitHub Actions [Install Test Workflow](../.github/workflows/install.yaml). ### Framework @@ -124,11 +126,6 @@ vagrant provision --provision-with=k3s-status vagrant provision --provision-with=k3s-procps ``` -The **Snapshotter** test requires that k3s binary is built at `dist/artifacts/k3s`. -It is invoked similarly, i.e. `vagrant up`, but with different sets of named shell provisioners. -Take a look at the individual Vagrantfiles and/or the Github Actions workflows that harness them to get -an idea of how they can be invoked. - ___ ## Performance Tests diff --git a/scripts/flaky-tests b/tests/docker/flaky-tests similarity index 100% rename from scripts/flaky-tests rename to tests/docker/flaky-tests diff --git a/scripts/test-helpers b/tests/docker/test-helpers similarity index 99% rename from scripts/test-helpers rename to tests/docker/test-helpers index c34cc3bda93f..6a5335689557 100755 --- a/scripts/test-helpers +++ b/tests/docker/test-helpers @@ -511,7 +511,7 @@ run-test() { ) export PROVISION_LOCK=$(mktemp) - ./scripts/test-runner "$@" & + ./tests/docker/test-runner "$@" & pids+=($!) ( @@ -585,7 +585,7 @@ test-run-sonobuoy() { fi cleanup-test-env - . ./scripts/test-setup-sonobuoy$suffix + . ./tests/docker/test-setup-sonobuoy$suffix if [ "$1" = "parallel" ] || [ "$2" = "parallel" ]; then label=PARALLEL \ diff --git a/scripts/test-run-basics b/tests/docker/test-run-basics similarity index 100% rename from scripts/test-run-basics rename to tests/docker/test-run-basics diff --git a/scripts/test-run-bootstraptoken b/tests/docker/test-run-bootstraptoken similarity index 100% rename from scripts/test-run-bootstraptoken rename to tests/docker/test-run-bootstraptoken diff --git a/scripts/test-run-cacerts b/tests/docker/test-run-cacerts similarity index 100% rename from scripts/test-run-cacerts rename to tests/docker/test-run-cacerts diff --git a/scripts/test-run-compat b/tests/docker/test-run-compat similarity index 100% rename from scripts/test-run-compat rename to tests/docker/test-run-compat diff --git a/scripts/test-run-etcd b/tests/docker/test-run-etcd similarity index 100% rename from scripts/test-run-etcd rename to tests/docker/test-run-etcd diff --git a/scripts/test-run-hardened b/tests/docker/test-run-hardened similarity index 100% rename from scripts/test-run-hardened rename to tests/docker/test-run-hardened diff --git a/scripts/test-run-lazypull b/tests/docker/test-run-lazypull similarity index 100% rename from scripts/test-run-lazypull rename to tests/docker/test-run-lazypull diff --git a/scripts/test-run-upgrade b/tests/docker/test-run-upgrade similarity index 100% rename from scripts/test-run-upgrade rename to tests/docker/test-run-upgrade diff --git a/scripts/test-runner b/tests/docker/test-runner similarity index 84% rename from scripts/test-runner rename to tests/docker/test-runner index 79b4ce10dae0..a831e74ea14d 100755 --- a/scripts/test-runner +++ b/tests/docker/test-runner @@ -1,6 +1,6 @@ #!/bin/bash set -x -e -cd $(dirname $0)/.. +cd $(dirname $0)/../.. # --- diff --git a/scripts/test-setup-sonobuoy b/tests/docker/test-setup-sonobuoy similarity index 91% rename from scripts/test-setup-sonobuoy rename to tests/docker/test-setup-sonobuoy index d86cc49a1b31..028ef1345de1 100755 --- a/scripts/test-setup-sonobuoy +++ b/tests/docker/test-setup-sonobuoy @@ -19,7 +19,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log") # Ignore sonobuoy failures if only these flaky tests have failed - flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) + flakyFails=$( grep -scF -f ./tests/docker/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-etcd b/tests/docker/test-setup-sonobuoy-etcd similarity index 86% rename from scripts/test-setup-sonobuoy-etcd rename to tests/docker/test-setup-sonobuoy-etcd index b90dec30da8c..1d3dae59d91f 100644 --- a/scripts/test-setup-sonobuoy-etcd +++ b/tests/docker/test-setup-sonobuoy-etcd @@ -1,6 +1,6 @@ #!/bin/bash -. ./scripts/test-setup-sonobuoy +. ./tests/docker/test-setup-sonobuoy export NUM_SERVERS=1 export NUM_AGENTS=1 @@ -20,7 +20,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log") # Ignore sonobuoy failures if only these flaky tests have failed - flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) + flakyFails=$( grep -scF -f ./tests/docker/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-mysql b/tests/docker/test-setup-sonobuoy-mysql similarity index 93% rename from scripts/test-setup-sonobuoy-mysql rename to tests/docker/test-setup-sonobuoy-mysql index c8ae42c0dbeb..fd5e98e3284b 100644 --- a/scripts/test-setup-sonobuoy-mysql +++ b/tests/docker/test-setup-sonobuoy-mysql @@ -1,6 +1,6 @@ #!/bin/bash -. ./scripts/test-setup-sonobuoy +. ./tests/docker/test-setup-sonobuoy # --- @@ -47,7 +47,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log") # Ignore sonobuoy failures if only these flaky tests have failed - flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) + flakyFails=$( grep -scF -f ./tests/docker/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/scripts/test-setup-sonobuoy-postgres b/tests/docker/test-setup-sonobuoy-postgres similarity index 93% rename from scripts/test-setup-sonobuoy-postgres rename to tests/docker/test-setup-sonobuoy-postgres index 4b7f794ed920..4290cb5ac7d5 100644 --- a/scripts/test-setup-sonobuoy-postgres +++ b/tests/docker/test-setup-sonobuoy-postgres @@ -1,6 +1,6 @@ #!/bin/bash -. ./scripts/test-setup-sonobuoy +. ./tests/docker/test-setup-sonobuoy # --- @@ -47,7 +47,7 @@ test-post-hook() { fi local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log") # Ignore sonobuoy failures if only these flaky tests have failed - flakyFails=$( grep -scF -f ./scripts/flaky-tests <<< "$failures" ) + flakyFails=$( grep -scF -f ./tests/docker/flaky-tests <<< "$failures" ) totalFails=$( grep -scF -e "[Fail]" <<< "$failures" ) [ "$totalFails" -le "$flakyFails" ] } diff --git a/tests/e2e/btrfs/Vagrantfile b/tests/e2e/btrfs/Vagrantfile new file mode 100644 index 000000000000..08b20c0f0b4c --- /dev/null +++ b/tests/e2e/btrfs/Vagrantfile @@ -0,0 +1,99 @@ +ENV['VAGRANT_NO_PARALLEL'] = 'no' +ENV['VAGRANT_EXPERIMENTAL']="disks" +ENV['VAGRANT_LOG']="error" +NODE_ROLES = (ENV['E2E_NODE_ROLES'] || + ["server-0"]) +NODE_BOXES = (ENV['E2E_NODE_BOXES'] || + ['opensuse/Leap-15.5.x86_64']) +GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") +RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") +NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i +NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i +# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks +NETWORK_PREFIX = "10.10.10" +install_type = "" + +def provision(vm, role, role_num, node_num) + vm.box = NODE_BOXES[node_num] + vm.hostname = role + # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 + vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" + + vagrant_defaults = '../vagrantdefaults.rb' + load vagrant_defaults if File.exists?(vagrant_defaults) + + defaultOSConfigure(vm) + install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) + + vm.provision 'setup-btrfs', type: 'shell', run: 'once', privileged: true do |sh| + sh.inline = <<~EOF + #!/usr/bin/env bash + set -eu -o pipefail + zypper install -y apparmor-parser btrfsprogs hostname + mkdir -p /var/lib/rancher/k3s /etc/rancher/k3s /usr/local/bin + if ! mountpoint -q /var/lib/rancher/k3s; then + : ${BTRFS_DEV:=#{ENV['BTRFS_DEV']}} + for disk in sd[b-d] vd[b-d] xd[b-d]; do + if [ -n "${BTRFS_DEV}" ]; then break; fi + : ${BTRFS_DEV:=$(test -b /dev/$disk && echo $disk)} + done + btrfs filesystem show /dev/${BTRFS_DEV:?unable to determine automatically, please specify} 2>/dev/null || mkfs -t btrfs /dev/${BTRFS_DEV} + mountpoint -q /mnt || mount -t btrfs /dev/${BTRFS_DEV} /mnt + btrfs subvolume show /mnt/@k3s 2>/dev/null || btrfs subvolume create /mnt/@k3s + umount /mnt + mount -t btrfs -o subvol=@k3s /dev/${BTRFS_DEV} /var/lib/rancher/k3s + fi + EOF + end + + if role.include?("server") && role_num == 0 + vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| + k3s.args = ["server"] + k3s.env = %W[#{install_type}] + k3s.config = <<~YAML + cluster-init: true + node-external-ip: "#{NETWORK_PREFIX}.100" + flannel-iface: eth1 + snapshotter: btrfs + token: vagrant + disable: + - local-storage + - metrics-server + - servicelb + - traefik + write-kubeconfig-mode: '0644' + YAML + end + end +end + +Vagrant.configure("2") do |config| + config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] + # Default provider is libvirt, virtualbox is only provided as a backup + config.vm.provider "libvirt" do |v| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + v.storage :file, :size => '8G' # Requires VAGRANT_EXPERIMENTAL="disks" + end + config.vm.provider "virtualbox" do |v,o| + v.cpus = NODE_CPUS + v.memory = NODE_MEMORY + v.gui = false + v.check_guest_additions = false + o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks" + end + + if NODE_ROLES.kind_of?(String) + NODE_ROLES = NODE_ROLES.split(" ", -1) + end + if NODE_BOXES.kind_of?(String) + NODE_BOXES = NODE_BOXES.split(" ", -1) + end + + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) + end + end +end diff --git a/tests/e2e/btrfs/btrfs_test.go b/tests/e2e/btrfs/btrfs_test.go new file mode 100644 index 000000000000..01bdb3384bed --- /dev/null +++ b/tests/e2e/btrfs/btrfs_test.go @@ -0,0 +1,97 @@ +package rotateca + +import ( + "flag" + "fmt" + "os" + "strings" + "testing" + + "github.com/k3s-io/k3s/tests/e2e" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ci = flag.Bool("ci", false, "running on CI") +var local = flag.Bool("local", false, "deploy a locally built K3s binary") + +// Environment Variables Info: +// E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master + +func Test_E2EBtrfsSnapshot(t *testing.T) { + RegisterFailHandler(Fail) + flag.Parse() + suiteConfig, reporterConfig := GinkgoConfiguration() + RunSpecs(t, "Btrfs Snapshot Test Suite", suiteConfig, reporterConfig) +} + +var ( + kubeConfigFile string + serverNodeNames []string +) + +var _ = ReportAfterEach(e2e.GenReport) + +var _ = Describe("Verify that btrfs based servers work", Ordered, func() { + Context("Btrfs Snapshots are taken", func() { + It("Starts up with no issues", func() { + var err error + // OS and server are hardcoded because only openSUSE Leap 15.5 natively supports Btrfs + if *local { + serverNodeNames, _, err = e2e.CreateLocalCluster("opensuse/Leap-15.5.x86_64", 1, 0) + } else { + serverNodeNames, _, err = e2e.CreateCluster("opensuse/Leap-15.5.x86_64", 1, 0) + } + Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) + fmt.Println("CLUSTER CONFIG") + fmt.Println("Server Nodes:", serverNodeNames) + kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks node and pod status", func() { + fmt.Printf("\nFetching node status\n") + Eventually(func(g Gomega) { + nodes, err := e2e.ParseNodes(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, node := range nodes { + g.Expect(node.Status).Should(Equal("Ready")) + } + }, "620s", "5s").Should(Succeed()) + _, _ = e2e.ParseNodes(kubeConfigFile, true) + + fmt.Printf("\nFetching pods status\n") + Eventually(func(g Gomega) { + pods, err := e2e.ParsePods(kubeConfigFile, false) + g.Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } + }, "620s", "5s").Should(Succeed()) + _, _ = e2e.ParsePods(kubeConfigFile, true) + }) + It("Checks that btrfs snapshots exist", func() { + cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs" + res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/\\d+")) + Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/\\d+")) + }) + }) +}) + +var failed bool +var _ = AfterEach(func() { + failed = failed || CurrentSpecReport().Failed() +}) + +var _ = AfterSuite(func() { + if !failed || *ci { + Expect(e2e.DestroyCluster()).To(Succeed()) + Expect(os.Remove(kubeConfigFile)).To(Succeed()) + } +}) diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile index 9987bc031822..c17efa0fa99b 100644 --- a/tests/e2e/dualstack/Vagrantfile +++ b/tests/e2e/dualstack/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -101,13 +101,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index daf8f212581e..1c1b81260f00 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") @@ -190,9 +190,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/embeddedmirror/Vagrantfile b/tests/e2e/embeddedmirror/Vagrantfile index 4d44de1ecb15..ab8750ebdd1e 100644 --- a/tests/e2e/embeddedmirror/Vagrantfile +++ b/tests/e2e/embeddedmirror/Vagrantfile @@ -1,8 +1,8 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) + ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") @@ -27,7 +27,6 @@ def provision(vm, role, role_num, node_num) addCoverageDir(vm, role, GOCOVER) install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - vm.provision "shell", inline: "ping -c 2 k3s.io" # The formatting on this is a little weird, but it allows inserting variables # and still using the heredoc formatting with escapped quotes @@ -112,13 +111,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/embeddedmirror/embeddedmirror_test.go b/tests/e2e/embeddedmirror/embeddedmirror_test.go index d38a0a51337a..c50923fbf80d 100644 --- a/tests/e2e/embeddedmirror/embeddedmirror_test.go +++ b/tests/e2e/embeddedmirror/embeddedmirror_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, +// generic/ubuntu2204, generic/centos7, generic/rocky8, // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") @@ -142,10 +142,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { r2, err := e2e.RunCmdOnNode("kubectl delete deployment my-webpage-1 my-webpage-2 my-webpage-3", serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), r2) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/externalip/Vagrantfile b/tests/e2e/externalip/Vagrantfile index c519615506fe..5e6181e4421f 100644 --- a/tests/e2e/externalip/Vagrantfile +++ b/tests/e2e/externalip/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -73,13 +73,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/externalip/externalip_test.go b/tests/e2e/externalip/externalip_test.go index d3fccb4da0ff..6163a03a784b 100644 --- a/tests/e2e/externalip/externalip_test.go +++ b/tests/e2e/externalip/externalip_test.go @@ -17,12 +17,13 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") var ci = flag.Bool("ci", false, "running on CI") +var local = flag.Bool("local", false, "deploy a locally built K3s binary") // getLBServiceIPs returns the externalIP configured for flannel func getExternalIPs(kubeConfigFile string) ([]string, error) { @@ -66,7 +67,11 @@ var _ = Describe("Verify External-IP config", Ordered, func() { It("Starts up with no issues", func() { var err error - serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + if *local { + serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) + } else { + serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) + } Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) fmt.Println("CLUSTER CONFIG") fmt.Println("OS:", *nodeOS) @@ -160,9 +165,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/privateregistry/Vagrantfile b/tests/e2e/privateregistry/Vagrantfile index f58f97c75dd9..7cb2af3aff70 100644 --- a/tests/e2e/privateregistry/Vagrantfile +++ b/tests/e2e/privateregistry/Vagrantfile @@ -1,8 +1,8 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || - ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) + ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -25,7 +25,6 @@ def provision(vm, role, role_num, node_num) defaultOSConfigure(vm) install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - vm.provision "shell", inline: "ping -c 2 k3s.io" # The formatting on this is a little weird, but it allows inserting variables # and still using the heredoc formatting with escapped quotes @@ -115,13 +114,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/privateregistry/privateregistry_test.go b/tests/e2e/privateregistry/privateregistry_test.go index 62e1e1480e2b..508070087f8c 100644 --- a/tests/e2e/privateregistry/privateregistry_test.go +++ b/tests/e2e/privateregistry/privateregistry_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, +// generic/ubuntu2204, generic/centos7, generic/rocky8, // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") @@ -146,10 +146,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { r1, err := e2e.RunCmdOnNode("docker rm -f registry", serverNodeNames[0]) Expect(err).NotTo(HaveOccurred(), r1) r2, err := e2e.RunCmdOnNode("kubectl delete deployment my-webpage", serverNodeNames[0]) diff --git a/tests/e2e/rotateca/Vagrantfile b/tests/e2e/rotateca/Vagrantfile index bb50599ca58f..c8a64729519f 100644 --- a/tests/e2e/rotateca/Vagrantfile +++ b/tests/e2e/rotateca/Vagrantfile @@ -76,13 +76,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/rotateca/rotateca_test.go b/tests/e2e/rotateca/rotateca_test.go index 29e187e80273..87d6aef70840 100644 --- a/tests/e2e/rotateca/rotateca_test.go +++ b/tests/e2e/rotateca/rotateca_test.go @@ -12,7 +12,7 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") @@ -133,9 +133,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/s3/Vagrantfile b/tests/e2e/s3/Vagrantfile index 858bf023ec96..984ec7cae9aa 100644 --- a/tests/e2e/s3/Vagrantfile +++ b/tests/e2e/s3/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004']) + ['generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -24,8 +24,6 @@ def provision(vm, role, role_num, node_num) defaultOSConfigure(vm) install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) - - vm.provision "shell", inline: "ping -c 2 k3s.io" runS3mock = <<~'SCRIPT' docker run -p 9090:9090 -p 9191:9191 -d -e initialBuckets=test -e debug=true -t adobe/s3mock @@ -80,13 +78,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/s3/s3_test.go b/tests/e2e/s3/s3_test.go index f24b2eb826de..a4a23e09a1de 100644 --- a/tests/e2e/s3/s3_test.go +++ b/tests/e2e/s3/s3_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, +// generic/ubuntu2204, generic/centos7, generic/rocky8, // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") @@ -117,21 +117,32 @@ var _ = Describe("Verify Create", Ordered, func() { It("delete first on-demand s3 snapshot", func() { _, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot ls >> ./snapshotname.txt", serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) - snapshotName, err := e2e.RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt | sed 's/^/on-demand-server-0-/'| head -1", serverNodeNames[0]) + snapshotName, err := e2e.RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt |head -1", serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) res, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot delete "+snapshotName, serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) - Expect(res).To(ContainSubstring("Removing the given etcd snapshot(s) from S3")) - Expect(res).To(ContainSubstring("Reconciliation of snapshot data in k3s-etcd-snapshots ConfigMap complete")) - Expect(res).To(ContainSubstring("Removing the given locally stored etcd snapshot(s)")) + Expect(res).To(ContainSubstring("Reconciling ETCDSnapshotFile resources")) + Expect(res).To(ContainSubstring("Snapshot " + strings.TrimSpace(snapshotName) + " deleted from S3")) + Expect(res).To(ContainSubstring("Reconciliation of ETCDSnapshotFile resources complete")) }) // TODO, there is currently a bug that prevents pruning on s3 snapshots that are not prefixed with "on-demand" // https://github.com/rancher/rke2/issues/3714 // Once fixed, ensure that the snapshots list are actually reduced to 2 It("prunes s3 snapshots", func() { - _, err := e2e.RunCmdOnNode("k3s etcd-snapshot prune --snapshot-retention 2", serverNodeNames[0]) + _, err := e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0]) Expect(err).NotTo(HaveOccurred()) + _, err = e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + res, err := e2e.RunCmdOnNode("k3s etcd-snapshot prune --snapshot-retention 2", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(ContainSubstring("Reconciliation of ETCDSnapshotFile resources complete")) + + _, err = e2e.RunCmdOnNode("k3s etcd-snapshot ls|grep 'on-demand'|wc -l>count", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + res, err = e2e.RunCmdOnNode("grep '^[4]$' ./count", serverNodeNames[0]) + Expect(err).NotTo(HaveOccurred()) + Expect(strings.TrimSpace(res)).To(Equal("4")) }) }) }) @@ -142,10 +153,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/scripts/run_tests.sh b/tests/e2e/scripts/run_tests.sh index aeb2c4127001..25bdcf283392 100755 --- a/tests/e2e/scripts/run_tests.sh +++ b/tests/e2e/scripts/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -nodeOS=${1:-"generic/ubuntu2004"} +nodeOS=${1:-"generic/ubuntu2204"} servercount=${2:-3} agentcount=${3:-1} db=${4:-"etcd"} diff --git a/tests/e2e/secretsencryption/Vagrantfile b/tests/e2e/secretsencryption/Vagrantfile index b98322e76156..07daf4286911 100644 --- a/tests/e2e/secretsencryption/Vagrantfile +++ b/tests/e2e/secretsencryption/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -62,13 +62,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index 5bc23d8dc0e8..fb8c6e4c9160 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var hardened = flag.Bool("hardened", false, "true or false") var ci = flag.Bool("ci", false, "running on CI") @@ -306,9 +306,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/snapshotrestore/Vagrantfile b/tests/e2e/snapshotrestore/Vagrantfile index 05cea6d814aa..fd0682e95d96 100644 --- a/tests/e2e/snapshotrestore/Vagrantfile +++ b/tests/e2e/snapshotrestore/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -94,13 +94,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/snapshotrestore/snapshotrestore_test.go b/tests/e2e/snapshotrestore/snapshotrestore_test.go index 37a88f95a640..4805d6cedc18 100644 --- a/tests/e2e/snapshotrestore/snapshotrestore_test.go +++ b/tests/e2e/snapshotrestore/snapshotrestore_test.go @@ -14,10 +14,10 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, +// generic/ubuntu2204, generic/centos7, generic/rocky8, // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") @@ -317,9 +317,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile index e88aaa01ef6e..c6c8671a6415 100644 --- a/tests/e2e/splitserver/Vagrantfile +++ b/tests/e2e/splitserver/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-etcd-0", "server-etcd-1", "server-etcd-2", "server-cp-0", "server-cp-1", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -115,13 +115,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index 70cf99ad073f..9fb80e91807e 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -13,8 +13,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var etcdCount = flag.Int("etcdCount", 3, "number of server nodes only deploying etcd") var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") @@ -229,9 +229,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/startup/Vagrantfile b/tests/e2e/startup/Vagrantfile index 37b441e0649d..ebad1f115e27 100644 --- a/tests/e2e/startup/Vagrantfile +++ b/tests/e2e/startup/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -25,8 +25,8 @@ def provision(vm, role, role_num, node_num) install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) node_ip = "#{NETWORK_PREFIX}.#{100+node_num}" - vm.provision "shell", inline: "ping -c 2 k3s.io" - + + if role.include?("server") vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| k3s.args = "server " @@ -76,13 +76,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index 4dd881aac6d4..590666565b41 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var ci = flag.Bool("ci", false, "running on CI") var local = flag.Bool("local", false, "deploy a locally built K3s binary") @@ -181,77 +181,6 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { Expect(err).NotTo(HaveOccurred()) }) }) - Context("Verify disable-agent and egress-selector-mode flags", func() { - It("Starts K3s with no issues", func() { - disableAgentYAML := "disable-agent: true\negress-selector-mode: cluster" - err := StartK3sCluster(append(serverNodeNames, agentNodeNames...), disableAgentYAML, "") - Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) - - fmt.Println("CLUSTER CONFIG") - fmt.Println("OS:", *nodeOS) - fmt.Println("Server Nodes:", serverNodeNames) - fmt.Println("Agent Nodes:", agentNodeNames) - kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Checks node and pod status", func() { - fmt.Printf("\nFetching node status\n") - Eventually(func(g Gomega) { - nodes, err := e2e.ParseNodes(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready")) - } - }, "360s", "5s").Should(Succeed()) - _, _ = e2e.ParseNodes(kubeConfigFile, true) - - fmt.Printf("\nFetching pods status\n") - Eventually(func(g Gomega) { - pods, err := e2e.ParsePods(kubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - } - } - }, "360s", "5s").Should(Succeed()) - _, _ = e2e.ParsePods(kubeConfigFile, true) - }) - - It("Returns pod metrics", func() { - cmd := "kubectl top pod -A" - Eventually(func() error { - _, err := e2e.RunCommand(cmd) - return err - }, "600s", "5s").Should(Succeed()) - }) - - It("Returns node metrics", func() { - cmd := "kubectl top node" - _, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Runs an interactive command a pod", func() { - cmd := "kubectl run busybox --rm -it --restart=Never --image=rancher/mirrored-library-busybox:1.36.1 -- uname -a" - _, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Collects logs from a pod", func() { - cmd := "kubectl logs -n kube-system -l app.kubernetes.io/name=traefik -c traefik" - _, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Kills the cluster", func() { - err := KillK3sCluster(append(serverNodeNames, agentNodeNames...)) - Expect(err).NotTo(HaveOccurred()) - }) - }) Context("Verify server fails to start with bootstrap token", func() { It("Fails to start with a meaningful error", func() { tokenYAML := "token: aaaaaa.bbbbbbbbbbbbbbbb" @@ -277,9 +206,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/tailscale/Vagrantfile b/tests/e2e/tailscale/Vagrantfile index d8c782adb90d..3c3df7299169 100644 --- a/tests/e2e/tailscale/Vagrantfile +++ b/tests/e2e/tailscale/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i @@ -71,14 +71,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - config.vm.define name do |node| - roles = name.split("-", -1) - role_num = roles.pop.to_i - provision(node.vm, roles, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/tailscale/tailscale_test.go b/tests/e2e/tailscale/tailscale_test.go index 156c67954816..1241f35eae0b 100644 --- a/tests/e2e/tailscale/tailscale_test.go +++ b/tests/e2e/tailscale/tailscale_test.go @@ -11,8 +11,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") @@ -117,9 +117,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 492b298891e7..c53961f3a1f0 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -111,11 +112,11 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri } // Bring up the first server node cmd := fmt.Sprintf(`%s %s vagrant up %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0]) - fmt.Println(cmd) if _, err := RunCommand(cmd); err != nil { return nil, nil, newNodeError(cmd, serverNodeNames[0], err) } + // Bring up the rest of the nodes in parallel errg, _ := errgroup.WithContext(context.Background()) for _, node := range append(serverNodeNames[1:], agentNodeNames...) { @@ -170,17 +171,21 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [ } testOptions += " E2E_RELEASE_VERSION=skip" - // Bring up the all of the nodes in parallel + // Provision the first server node. In GitHub Actions, this also imports the VM image into libvirt, which + // takes time and can cause the next vagrant up to fail if it is not given enough time to complete. + cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, serverNodeNames[0]) + fmt.Println(cmd) + if _, err := RunCommand(cmd); err != nil { + return nil, nil, newNodeError(cmd, serverNodeNames[0], err) + } + + // Bring up the rest of the nodes in parallel errg, _ := errgroup.WithContext(context.Background()) - for i, node := range append(serverNodeNames, agentNodeNames...) { - if i == 0 { - cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &> vagrant.log`, nodeEnvs, testOptions, node) - } else { - cmd = fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node) - } + for _, node := range append(serverNodeNames[1:], agentNodeNames...) { + cmd := fmt.Sprintf(`%s %s vagrant up --no-provision %s &>> vagrant.log`, nodeEnvs, testOptions, node) errg.Go(func() error { if _, err := RunCommand(cmd); err != nil { - return fmt.Errorf("failed initializing nodes: %s: %v", cmd, err) + return newNodeError(cmd, node, err) } return nil }) @@ -190,10 +195,10 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [ if err := errg.Wait(); err != nil { return nil, nil, err } + if err := scpK3sBinary(append(serverNodeNames, agentNodeNames...)); err != nil { return nil, nil, err } - // Install K3s on all nodes in parallel errg, _ = errgroup.WithContext(context.Background()) for _, node := range append(serverNodeNames, agentNodeNames...) { @@ -285,6 +290,8 @@ func GenKubeConfigFile(serverName string) (string, error) { if err != nil { return "", err } + re := regexp.MustCompile(`(?m)==> vagrant:.*\n`) + kubeConfig = re.ReplaceAllString(kubeConfig, "") nodeIP, err := FetchNodeExternalIP(serverName) if err != nil { return "", err @@ -467,6 +474,9 @@ func RunCommand(cmd string) (string, error) { c.Env = append(os.Environ(), "KUBECONFIG="+kc) } out, err := c.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("failed to run command: %s, %v", cmd, err) + } return string(out), err } diff --git a/tests/e2e/token/Vagrantfile b/tests/e2e/token/Vagrantfile index 0a61b7ac57e3..b137c1239b3f 100644 --- a/tests/e2e/token/Vagrantfile +++ b/tests/e2e/token/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") @@ -81,14 +81,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - config.vm.define name do |node| - roles = name.split("-", -1) - role_num = roles.pop.to_i - provision(node.vm, roles, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 387c15dbf13f..58d3ad3f52d8 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -14,9 +14,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, opensuse/Leap-15.5.x86_64 +// generic/ubuntu2204, generic/centos7, generic/rocky8, opensuse/Leap-15.5.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") @@ -201,9 +201,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/upgradecluster/Vagrantfile b/tests/e2e/upgradecluster/Vagrantfile index 149564d8c556..f680667b2fca 100644 --- a/tests/e2e/upgradecluster/Vagrantfile +++ b/tests/e2e/upgradecluster/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) RELEASE_CHANNEL = (ENV['E2E_RELEASE_CHANNEL'] || "latest") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") @@ -154,13 +154,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index d4acd7d6b027..13c316f69738 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -13,9 +13,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8 +// generic/ubuntu2204, generic/centos7, generic/rocky8 // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") @@ -385,9 +385,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/validatecluster/Vagrantfile b/tests/e2e/validatecluster/Vagrantfile index 18409d3baf79..94b1286f6b6e 100644 --- a/tests/e2e/validatecluster/Vagrantfile +++ b/tests/e2e/validatecluster/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "server-1", "server-2", "agent-0", "agent-1"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004']) + ['generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204', 'generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") EXTERNAL_DB = (ENV['E2E_EXTERNAL_DB'] || "etcd") @@ -151,13 +151,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 0ce71fffd064..3b91c968143d 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -14,9 +14,9 @@ import ( ) // Valid nodeOS: -// generic/ubuntu2004, generic/centos7, generic/rocky8, +// generic/ubuntu2204, generic/centos7, generic/rocky8, // opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 3, "number of server nodes") var agentCount = flag.Int("agentCount", 2, "number of agent nodes") var hardened = flag.Bool("hardened", false, "true or false") @@ -382,9 +382,7 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") - } else { + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/e2e/wasm/Vagrantfile b/tests/e2e/wasm/Vagrantfile index 055ccf185d71..fac00c9a0397 100644 --- a/tests/e2e/wasm/Vagrantfile +++ b/tests/e2e/wasm/Vagrantfile @@ -2,7 +2,7 @@ ENV['VAGRANT_NO_PARALLEL'] = 'no' NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0"]) NODE_BOXES = (ENV['E2E_NODE_BOXES'] || - ['generic/ubuntu2004']) + ['generic/ubuntu2204']) GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") GOCOVER = (ENV['E2E_GOCOVER'] || "") @@ -84,13 +84,10 @@ Vagrant.configure("2") do |config| NODE_BOXES = NODE_BOXES.split(" ", -1) end - # Must iterate on the index, vagrant does not understand iterating - # over the node roles themselves - NODE_ROLES.length.times do |i| - name = NODE_ROLES[i] - role_num = name.split("-", -1).pop.to_i - config.vm.define name do |node| - provision(node.vm, name, role_num, i) + NODE_ROLES.each_with_index do |role, i| + role_num = role.split("-", -1).pop.to_i + config.vm.define role do |node| + provision(node.vm, role, role_num, i) end end end diff --git a/tests/e2e/wasm/wasm_test.go b/tests/e2e/wasm/wasm_test.go index 3cb57327f1ee..01a1ed6f3b43 100644 --- a/tests/e2e/wasm/wasm_test.go +++ b/tests/e2e/wasm/wasm_test.go @@ -12,8 +12,8 @@ import ( . "github.com/onsi/gomega" ) -// Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64 -var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system") +// Valid nodeOS: generic/ubuntu2204, opensuse/Leap-15.3.x86_64 +var nodeOS = flag.String("nodeOS", "generic/ubuntu2204", "VM operating system") var serverCount = flag.Int("serverCount", 1, "number of server nodes") var agentCount = flag.Int("agentCount", 0, "number of agent nodes") var ci = flag.Bool("ci", false, "running on CI") @@ -138,7 +138,6 @@ var _ = AfterSuite(func() { if failed && !*ci { fmt.Println("FAILED!") } else { - Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) } diff --git a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile b/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile deleted file mode 100644 index 167a448de6d6..000000000000 --- a/tests/snapshotter/btrfs/opensuse-leap/Vagrantfile +++ /dev/null @@ -1,121 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : -# -# Vagrant box for testing k3s with the btrfs snapshotter. Usage: -# VAGRANT_EXPERIMENTAL=disks vagrant up - -Vagrant.configure("2") do |config| - config.vagrant.plugins = ["vagrant-k3s"] - config.vm.box = "opensuse/Leap-15.5.x86_64" - config.vm.boot_timeout = ENV['TEST_VM_BOOT_TIMEOUT'] || 600 # seconds - config.vm.synced_folder '../../../../dist/artifacts', '/vagrant', type: 'rsync', disabled: false, - rsync__exclude: ENV['RSYNC_EXCLUDE'] || '*.tar.*' - - config.vm.define 'snapshotter-btrfs', primary: true do |test| - test.vm.hostname = 'smoke' - test.vm.provision 'k3s-prepare', type: 'shell', run: 'once', privileged: true do |sh| - sh.inline = <<~EOF - #!/usr/bin/env bash - set -eu -o pipefail - zypper install -y apparmor-parser btrfsprogs hostname - mkdir -p /var/lib/rancher/k3s /etc/rancher/k3s /usr/local/bin - if ! mountpoint -q /var/lib/rancher/k3s; then - : ${BTRFS_DEV:=#{ENV['BTRFS_DEV']}} - for disk in sd[b-d] vd[b-d] xd[b-d]; do - if [ -n "${BTRFS_DEV}" ]; then break; fi - : ${BTRFS_DEV:=$(test -b /dev/$disk && echo $disk)} - done - btrfs filesystem show /dev/${BTRFS_DEV:?unable to determine automatically, please specify} 2>/dev/null || mkfs -t btrfs /dev/${BTRFS_DEV} - mountpoint -q /mnt || mount -t btrfs /dev/${BTRFS_DEV} /mnt - btrfs subvolume show /mnt/@k3s 2>/dev/null || btrfs subvolume create /mnt/@k3s - umount /mnt - mount -t btrfs -o subvol=@k3s /dev/${BTRFS_DEV} /var/lib/rancher/k3s - fi - # Install k3s binary - install -m 755 /vagrant/k3s /usr/local/bin - if [ -e /vagrant/*.tar ]; then - mkdir -vp /var/lib/rancher/k3s/agent/images - for tar in /vagrant/*.tar; do - cp -vf $tar /var/lib/rancher/k3s/agent/images/ - done - fi - EOF - end - test.vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| - k3s.args = %w[server --snapshotter=btrfs] - k3s.env = %w[INSTALL_K3S_NAME=server INSTALL_K3S_SKIP_DOWNLOAD=true K3S_TOKEN=vagrant] - k3s.config = <<~YAML - disable: - - local-storage - - metrics-server - - servicelb - - traefik - disable-helm-controller: true - disable-network-policy: true - write-kubeconfig-mode: '0644' - YAML - k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321 - end - test.vm.provision "k3s-wait-for-node", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - echo 'Waiting for node to be ready ...' - time timeout 300 bash -c 'while ! (kubectl wait --for condition=ready node/$(hostnamectl --static) 2>/dev/null); do sleep 5; done' - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-wait-for-coredns", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eu -o pipefail - function describe-coredns { - RC=$? - if [[ $RC -ne 0 ]]; then - kubectl describe node - kubectl --namespace kube-system describe pod -l k8s-app=kube-dns - kubectl --namespace kube-system logs -l k8s-app=kube-dns - fi - exit $RC - } - trap describe-coredns EXIT - time timeout 300 bash -c 'while ! (kubectl --namespace kube-system rollout status --timeout 10s deploy/coredns 2>/dev/null); do sleep 5; done' - SHELL - end - test.vm.provision "k3s-status", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - kubectl get node,all -A -o wide - SHELL - end - test.vm.provision "k3s-snapshots", type: "shell", run: ENV['CI'] == 'true' ? 'never' : 'once' do |sh| - sh.env = { :PATH => "/usr/local/bin:/usr/local/sbin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin" } - sh.inline = <<~SHELL - #!/usr/bin/env bash - set -eux -o pipefail - btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs - SHELL - end - end - - %w[libvirt virtualbox].each do |p| - config.vm.provider p do |v| - v.cpus = ENV['TEST_VM_CPUS'] || 2 - v.memory = ENV['TEST_VM_MEMORY'] || 2048 - end - end - - config.vm.provider :libvirt do |v,o| - v.storage :file, :size => '8G' - end - - config.vm.provider :virtualbox do |v,o| - v.gui = false - v.check_guest_additions = false - o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks" - end -end