diff --git a/.github/actions/vagrant-setup/action.yaml b/.github/actions/vagrant-setup/action.yaml new file mode 100644 index 000000000000..28a8f2b0bb5f --- /dev/null +++ b/.github/actions/vagrant-setup/action.yaml @@ -0,0 +1,25 @@ +name: 'Setup Vagrant and Libvirt' +description: 'A composite action that installs latest versions of vagrant and libvirt for use on ubuntu based runners' +runs: + using: 'composite' + steps: + - name: Add vagrant to apt-get sources + shell: bash + run: | + curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list + sudo sed -i 's/^# deb-src/deb-src/' /etc/apt/sources.list + - name: Install vagrant and libvirt + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libvirt-daemon libvirt-daemon-system vagrant + sudo systemctl enable --now libvirtd + - name: Build vagrant dependencies + shell: bash + run: | + sudo apt-get build-dep -y vagrant ruby-libvirt + sudo apt-get install -y --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev + - name: Install vagrant-libvirt plugin + shell: bash + run: sudo vagrant plugin install vagrant-libvirt \ No newline at end of file diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index 71a86a5e66fc..5f82a4477346 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -25,13 +25,13 @@ jobs: test: name: "Smoke Test" needs: build - runs-on: macos-12 + runs-on: ubuntu-latest timeout-minutes: 40 strategy: fail-fast: false matrix: vm: [centos-7, rocky-8, rocky-9, fedora, opensuse-leap, ubuntu-2204] - max-parallel: 2 + max-parallel: 3 defaults: run: working-directory: tests/install/${{ matrix.vm }} @@ -41,52 +41,70 @@ jobs: - name: "Checkout" uses: actions/checkout@v4 with: {fetch-depth: 1} + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup + - name: "Vagrant Cache" - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems + /tmp/boxes key: vagrant-box-${{ matrix.vm }} - id: vagrant-cache - continue-on-error: true + # Workaround for https://github.com/actions/cache/issues/1319 + - name: Move vagrant cache to /root + run: | + mkdir -p /tmp/boxes + sudo rm -rf /root/.vagrant.d/boxes + sudo mv -f /tmp/boxes /root/.vagrant.d - name: "Vagrant Plugin(s)" - run: vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp + run: sudo vagrant plugin install vagrant-k3s vagrant-reload vagrant-scp - name: "Download k3s binary" uses: actions/download-artifact@v3 with: name: k3s path: tests/install/${{ matrix.vm }} - name: "Vagrant Up" - run: vagrant up --no-provision - - name: "Upload k3s binary" + run: sudo vagrant up --no-provision + - name: "Upload k3s binary to VM" run: | chmod +x k3s - vagrant scp k3s /tmp/k3s - vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" - vagrant provision --provision-with=k3s-upload + sudo vagrant scp k3s /tmp/k3s + sudo vagrant ssh -c "sudo mv /tmp/k3s /usr/local/bin/k3s" + sudo vagrant provision --provision-with=k3s-upload - name: Add binary to PATH if: matrix.vm == 'centos-7' || matrix.vm == 'rocky-8' || matrix.vm == 'rocky-9' || matrix.vm == 'opensuse-leap' - run: vagrant provision --provision-with=add-bin-path + run: sudo vagrant provision --provision-with=add-bin-path - name: "⏩ Install K3s" run: | - vagrant provision --provision-with=k3s-prepare - vagrant provision --provision-with=k3s-install + sudo vagrant provision --provision-with=k3s-prepare + sudo vagrant provision --provision-with=k3s-install if [ ${{ matrix.vm }} = 'opensuse-microos' ]; then vagrant reload --no-provision; fi - name: "⏳ Node" - run: vagrant provision --provision-with=k3s-wait-for-node + run: sudo vagrant provision --provision-with=k3s-wait-for-node - name: "⏳ CoreDNS" - run: vagrant provision --provision-with=k3s-wait-for-coredns + run: sudo vagrant provision --provision-with=k3s-wait-for-coredns - name: "⏳ Local Storage" - run: vagrant provision --provision-with=k3s-wait-for-local-storage + run: sudo vagrant provision --provision-with=k3s-wait-for-local-storage continue-on-error: true - name: "⏳ Metrics Server" - run: vagrant provision --provision-with=k3s-wait-for-metrics-server + run: sudo vagrant provision --provision-with=k3s-wait-for-metrics-server continue-on-error: true - name: "⏳ Traefik" - run: vagrant provision --provision-with=k3s-wait-for-traefik + run: sudo vagrant provision --provision-with=k3s-wait-for-traefik continue-on-error: true - name: "k3s-status" - run: vagrant provision --provision-with=k3s-status + run: sudo vagrant provision --provision-with=k3s-status - name: "k3s-procps" - run: vagrant provision --provision-with=k3s-procps + run: sudo vagrant provision --provision-with=k3s-procps + - name: Cleanup VM + run: sudo vagrant destroy -f + - name: On Failure, launch debug session + uses: lhotari/action-upterm@v1 + if: ${{ failure() }} + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 + - name: Copy out vagrant boxes for cache + run: | + sudo mv -f /root/.vagrant.d/boxes /tmp/boxes + sudo chmod -R 777 /tmp/boxes \ No newline at end of file diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 383c53d6f819..a871537ac841 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -65,9 +65,11 @@ jobs: mkdir -p $GOCOVERDIR sudo -E env "PATH=$PATH" go test -v -timeout=45m ./tests/integration/${{ matrix.itest }}/... -run Integration - name: On Failure, Launch Debug Session + uses: lhotari/action-upterm@v1 if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - timeout-minutes: 5 + with: + ## If no one connects after 5 minutes, shut down server. + wait-timeout-minutes: 5 - name: Generate coverage report run: go tool covdata textfmt -i $GOCOVERDIR -o ${{ matrix.itest }}.out - name: Upload Results To Codecov diff --git a/.github/workflows/nightly-install.yaml b/.github/workflows/nightly-install.yaml index 417219c62329..99d0c067ea77 100644 --- a/.github/workflows/nightly-install.yaml +++ b/.github/workflows/nightly-install.yaml @@ -10,7 +10,7 @@ permissions: jobs: test: name: "Smoke Test" - runs-on: macos-12 + runs-on: ubuntu-latest timeout-minutes: 40 strategy: fail-fast: false @@ -27,15 +27,21 @@ jobs: - name: "Checkout" uses: actions/checkout@v4 with: {fetch-depth: 1} + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup + # Workaround for https://github.com/actions/cache/issues/1319 + - name: Move vagrant cache to /root + run: | + mkdir -p /tmp/boxes + sudo rm -rf /root/.vagrant.d/boxes + sudo mv -f /tmp/boxes /root/.vagrant.d - name: "Vagrant Cache" - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems + /tmp/boxes key: vagrant-box-${{ matrix.vm }} id: vagrant-cache - continue-on-error: true - name: "Vagrant Plugin(s)" run: vagrant plugin install vagrant-k3s vagrant-reload - name: "Vagrant Up ⏩ Install K3s" @@ -57,3 +63,7 @@ jobs: run: vagrant provision --provision-with=k3s-status - name: "k3s-procps" run: vagrant provision --provision-with=k3s-procps + - name: Copy out vagrant boxes for cache + run: | + sudo mv -f /root/.vagrant.d/boxes /tmp/boxes + sudo chmod -R 777 /tmp/boxes \ No newline at end of file diff --git a/.github/workflows/snapshotter.yaml b/.github/workflows/snapshotter.yaml index 396c8bd82545..c9e7805c2637 100644 --- a/.github/workflows/snapshotter.yaml +++ b/.github/workflows/snapshotter.yaml @@ -29,8 +29,7 @@ jobs: test: name: "Smoke Test" needs: build - # nested virtualization is only available on macOS hosts - runs-on: macos-12 + runs-on: ubuntu-latest timeout-minutes: 40 strategy: fail-fast: false @@ -50,24 +49,34 @@ jobs: - name: "Download Binary" uses: actions/download-artifact@v3 with: { name: k3s, path: dist/artifacts/ } + - name: Set up vagrant and libvirt + uses: ./.github/actions/vagrant-setup + # Workaround for https://github.com/actions/cache/issues/1319 + - name: Move vagrant cache to /root + run: | + mkdir -p /tmp/boxes + sudo rm -rf /root/.vagrant.d/boxes + sudo mv -f /tmp/boxes /root/.vagrant.d - name: "Vagrant Cache" - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | - ~/.vagrant.d/boxes - ~/.vagrant.d/gems + /tmp/boxes + /tmp/gems key: vagrant-box-${{ matrix.vm }} - id: vagrant-cache - continue-on-error: true - name: "Vagrant Plugin(s)" - run: vagrant plugin install vagrant-k3s + run: sudo vagrant plugin install vagrant-k3s - name: "Vagrant Up ⏩ Install K3s" - run: vagrant up + run: sudo vagrant up - name: "⏳ Node" - run: vagrant provision --provision-with=k3s-wait-for-node + run: sudo vagrant provision --provision-with=k3s-wait-for-node - name: "⏳ CoreDNS" - run: vagrant provision --provision-with=k3s-wait-for-coredns + run: sudo vagrant provision --provision-with=k3s-wait-for-coredns - name: "k3s-status" # kubectl get node,all -A -o wide - run: vagrant provision --provision-with=k3s-status + run: sudo vagrant provision --provision-with=k3s-status - name: "k3s-snapshots" # if no snapshots then we fail - run: vagrant provision --provision-with=k3s-snapshots + run: sudo vagrant provision --provision-with=k3s-snapshots + - name: Copy out vagrant boxes for cache + run: | + sudo mv -f /root/.vagrant.d/boxes /tmp/boxes + sudo chmod -R 777 /tmp/boxes \ No newline at end of file diff --git a/tests/TESTING.md b/tests/TESTING.md index 467d9235f03b..4a1029e93d41 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -61,28 +61,25 @@ ___ ## Smoke Tests -Smoke tests are a collection of tests defined under the [tests](../tests) path at the root of this repository. -The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path" scenarios. These fixtures are mostly self-contained Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers: - -- [Install Script](../tests/install) :arrow_right: scheduled nightly - - [CentOS 7](../tests/install/centos-7) (stand-in for RHEL 7) - - [Rocky Linux 8](../tests/install/rocky-8) (stand-in for RHEL 8) - - [Fedora 37](../tests/install/fedora) - - [Leap 15.4](../tests/install/opensuse-leap) (stand-in for SLES) - - [MicroOS](../tests/install/opensuse-microos) (stand-in for SLE-Micro) - - [Ubuntu 20.04](../tests/install/ubuntu-focal) (Focal Fossa) -- [Control Groups](../tests/cgroup) :arrow_right: on any code change - - [mode=unified](../tests/cgroup/unified) (cgroups v2) - - [Fedora 37](../tests/cgroup/unified/fedora) (rootfull + rootless) -- [Snapshotter](../tests/snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change - - [BTRFS](../tests/snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs)) +Smoke tests are a collection of tests defined under the [tests](./tests) and fall into two categories: install and snapshotter. These tests are used to validate the installation and operation of K3s on a variety of operating systems. The sub-directories therein contain fixtures for running simple clusters to assert correct behavior for "happy path" scenarios. The test themses are Vagrantfiles describing single-node installations that are easily spun up with Vagrant for the `libvirt` and `virtualbox` providers: + +- [Install Script](install) :arrow_right: scheduled nightly and on an install script change + - [CentOS 7](install/centos-7) (stand-in for RHEL 7) + - [Rocky Linux 8](install/rocky-8) (stand-in for RHEL 8) + - [Rocky Linux 9](install/rocky-9) (stand-in for RHEL 9) + - [Fedora 37](install/fedora) + - [Leap 15.5](install/opensuse-leap) (stand-in for SLES) + - [Ubuntu 22.04](install/ubuntu-2204) +- [Snapshotter](snapshotter/btrfs/opensuse-leap) :arrow_right: on any code change + - [BTRFS](snapshotter/btrfs) ([containerd built-in](https://github.com/containerd/containerd/tree/main/snapshots/btrfs)) - [Leap 15.4](../tests/snapshotter/btrfs/opensuse-leap) +## Format When adding new installer test(s) please copy the prevalent style for the `Vagrantfile`. -Ideally, the boxes used for additional assertions will support the default `virtualbox` provider which +Ideally, the boxes used for additional assertions will support the default `libvirt` provider which enables them to be used by our Github Actions Workflow(s). See: -- [cgroup.yaml](../.github/workflows/cgroup.yaml). - [install.yaml](../.github/workflows/install.yaml). +- [snapshotter.yaml](../.github/workflows/snapshotter.yaml). ### Framework @@ -92,11 +89,14 @@ If you are new to Vagrant, Hashicorp has written some pretty decent introductory #### Plugins and Providers -The `libvirt` and `vmware_desktop` providers cannot be used without first [installing the relevant plugins](https://www.vagrantup.com/docs/cli/plugin#plugin-install) -which are [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) and -[`vagrant-vmware-desktop`](https://www.vagrantup.com/docs/providers/vmware/installation), respectively. -Much like the default [`virtualbox` provider](https://www.vagrantup.com/docs/providers/virtualbox) these will do -nothing useful without also installing the relevant server runtimes and/or client programs. +The `libvirt`provider cannot be used without first [installing the `vagrant-libvirt` plugin](https://github.com/vagrant-libvirt/vagrant-libvirt). Libvirtd service must be installed and running on the host machine as well. + +Additionally, the `vagrant-scp` and `vagrant-k3s` plugins are required. + +All three can be and can be installed with: +```shell +vagrant plugin install vagrant-scp vagrant-k3s vagrant-libvirt +``` #### Environment Variables @@ -114,8 +114,8 @@ The **Install Script** tests can be run by changing to the fixture directory and ```shell cd tests/install/rocky-8 vagrant up -# the following provisioners are optional. the do not run by default but are invoked -# explicitly by github actions workflow to avoid certain timeout issues on slow runners +# The following provisioners are optional. In GitHub Actions CI they are invoked +# explicitly to avoid certain timeout issues on slow runners vagrant provision --provision-with=k3s-wait-for-node vagrant provision --provision-with=k3s-wait-for-coredns vagrant provision --provision-with=k3s-wait-for-local-storage @@ -125,8 +125,8 @@ vagrant provision --provision-with=k3s-status vagrant provision --provision-with=k3s-procps ``` -The **Control Groups** and **Snapshotter** tests require that k3s binary is built at `dist/artifacts/k3s`. -They are invoked similarly, i.e. `vagrant up`, but with different sets of named shell provisioners. +The **Snapshotter** test requires that k3s binary is built at `dist/artifacts/k3s`. +It is invoked similarly, i.e. `vagrant up`, but with different sets of named shell provisioners. Take a look at the individual Vagrantfiles and/or the Github Actions workflows that harness them to get an idea of how they can be invoked.