forked from k3s-io/k3s
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Convert snapshotter test in e2e test
Signed-off-by: Derek Nola <derek.nola@suse.com>
- Loading branch information
Showing
7 changed files
with
208 additions
and
217 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| ENV['VAGRANT_NO_PARALLEL'] = 'no' | ||
| ENV['VAGRANT_EXPERIMENTAL']="disks" | ||
| ENV['VAGRANT_LOG']="error" | ||
| NODE_ROLES = (ENV['E2E_NODE_ROLES'] || | ||
| ["server-0"]) | ||
| NODE_BOXES = (ENV['E2E_NODE_BOXES'] || | ||
| ['opensuse/Leap-15.5.x86_64']) | ||
| GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master") | ||
| RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "") | ||
| NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i | ||
| NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i | ||
| # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks | ||
| NETWORK_PREFIX = "10.10.10" | ||
| install_type = "" | ||
|
|
||
| def provision(vm, role, role_num, node_num) | ||
| vm.box = NODE_BOXES[node_num] | ||
| vm.hostname = role | ||
| # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32 | ||
| vm.network "private_network", ip: "#{NETWORK_PREFIX}.#{100+node_num}", netmask: "255.255.255.0" | ||
|
|
||
| vagrant_defaults = '../vagrantdefaults.rb' | ||
| load vagrant_defaults if File.exists?(vagrant_defaults) | ||
|
|
||
| defaultOSConfigure(vm) | ||
| install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH) | ||
|
|
||
| vm.provision 'setup-btrfs', type: 'shell', run: 'once', privileged: true do |sh| | ||
| sh.inline = <<~EOF | ||
| #!/usr/bin/env bash | ||
| set -eu -o pipefail | ||
| zypper install -y apparmor-parser btrfsprogs hostname | ||
| mkdir -p /var/lib/rancher/k3s /etc/rancher/k3s /usr/local/bin | ||
| if ! mountpoint -q /var/lib/rancher/k3s; then | ||
| : ${BTRFS_DEV:=#{ENV['BTRFS_DEV']}} | ||
| for disk in sd[b-d] vd[b-d] xd[b-d]; do | ||
| if [ -n "${BTRFS_DEV}" ]; then break; fi | ||
| : ${BTRFS_DEV:=$(test -b /dev/$disk && echo $disk)} | ||
| done | ||
| btrfs filesystem show /dev/${BTRFS_DEV:?unable to determine automatically, please specify} 2>/dev/null || mkfs -t btrfs /dev/${BTRFS_DEV} | ||
| mountpoint -q /mnt || mount -t btrfs /dev/${BTRFS_DEV} /mnt | ||
| btrfs subvolume show /mnt/@k3s 2>/dev/null || btrfs subvolume create /mnt/@k3s | ||
| umount /mnt | ||
| mount -t btrfs -o subvol=@k3s /dev/${BTRFS_DEV} /var/lib/rancher/k3s | ||
| fi | ||
| EOF | ||
| end | ||
|
|
||
| if role.include?("server") && role_num == 0 | ||
| vm.provision 'k3s-install', type: 'k3s', run: 'once' do |k3s| | ||
| k3s.args = ["server"] | ||
| k3s.env = %W[#{install_type}] | ||
| k3s.config = <<~YAML | ||
| cluster-init: true | ||
| node-external-ip: "#{NETWORK_PREFIX}.100" | ||
| flannel-iface: eth1 | ||
| snapshotter: btrfs | ||
| token: vagrant | ||
| disable: | ||
| - local-storage | ||
| - metrics-server | ||
| - servicelb | ||
| - traefik | ||
| write-kubeconfig-mode: '0644' | ||
| YAML | ||
| end | ||
| end | ||
| end | ||
|
|
||
| Vagrant.configure("2") do |config| | ||
| config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"] | ||
| # Default provider is libvirt, virtualbox is only provided as a backup | ||
| config.vm.provider "libvirt" do |v| | ||
| v.cpus = NODE_CPUS | ||
| v.memory = NODE_MEMORY | ||
| v.storage :file, :size => '8G' # Requires VAGRANT_EXPERIMENTAL="disks" | ||
| end | ||
| config.vm.provider "virtualbox" do |v,o| | ||
| v.cpus = NODE_CPUS | ||
| v.memory = NODE_MEMORY | ||
| v.gui = false | ||
| v.check_guest_additions = false | ||
| o.vm.disk :disk, name: "btrfs", size: "8GB" # Requires VAGRANT_EXPERIMENTAL="disks" | ||
| end | ||
|
|
||
| if NODE_ROLES.kind_of?(String) | ||
| NODE_ROLES = NODE_ROLES.split(" ", -1) | ||
| end | ||
| if NODE_BOXES.kind_of?(String) | ||
| NODE_BOXES = NODE_BOXES.split(" ", -1) | ||
| end | ||
|
|
||
| # Must iterate on the index, vagrant does not understand iterating | ||
| # over the node roles themselves | ||
| NODE_ROLES.length.times do |i| | ||
| name = NODE_ROLES[i] | ||
| role_num = name.split("-", -1).pop.to_i | ||
| config.vm.define name do |node| | ||
| provision(node.vm, name, role_num, i) | ||
| end | ||
| end | ||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package rotateca | ||
|
|
||
| import ( | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/k3s-io/k3s/tests/e2e" | ||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| ) | ||
|
|
||
| var ci = flag.Bool("ci", false, "running on CI") | ||
| var local = flag.Bool("local", false, "deploy a locally built K3s binary") | ||
|
|
||
| // Environment Variables Info: | ||
| // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master | ||
|
|
||
| func Test_E2EBtrfsSnapshot(t *testing.T) { | ||
| RegisterFailHandler(Fail) | ||
| flag.Parse() | ||
| suiteConfig, reporterConfig := GinkgoConfiguration() | ||
| RunSpecs(t, "Btrfs Snapshot Test Suite", suiteConfig, reporterConfig) | ||
| } | ||
|
|
||
| var ( | ||
| kubeConfigFile string | ||
| serverNodeNames []string | ||
| ) | ||
|
|
||
| var _ = ReportAfterEach(e2e.GenReport) | ||
|
|
||
| var _ = Describe("Verify that btrfs based servers work", Ordered, func() { | ||
| Context("Btrfs Snapshots are taken", func() { | ||
| It("Starts up with no issues", func() { | ||
| var err error | ||
| // OS and server are hardcoded because only openSUSE Leap 15.5 natively supports Btrfs | ||
| if *local { | ||
| serverNodeNames, _, err = e2e.CreateLocalCluster("opensuse/Leap-15.5.x86_64", 1, 0) | ||
| } else { | ||
| serverNodeNames, _, err = e2e.CreateCluster("opensuse/Leap-15.5.x86_64", 1, 0) | ||
| } | ||
| Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) | ||
| fmt.Println("CLUSTER CONFIG") | ||
| fmt.Println("Server Nodes:", serverNodeNames) | ||
| kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| }) | ||
|
|
||
| It("Checks node and pod status", func() { | ||
| fmt.Printf("\nFetching node status\n") | ||
| Eventually(func(g Gomega) { | ||
| nodes, err := e2e.ParseNodes(kubeConfigFile, false) | ||
| g.Expect(err).NotTo(HaveOccurred()) | ||
| for _, node := range nodes { | ||
| g.Expect(node.Status).Should(Equal("Ready")) | ||
| } | ||
| }, "620s", "5s").Should(Succeed()) | ||
| _, _ = e2e.ParseNodes(kubeConfigFile, true) | ||
|
|
||
| fmt.Printf("\nFetching pods status\n") | ||
| Eventually(func(g Gomega) { | ||
| pods, err := e2e.ParsePods(kubeConfigFile, false) | ||
| g.Expect(err).NotTo(HaveOccurred()) | ||
| for _, pod := range pods { | ||
| if strings.Contains(pod.Name, "helm-install") { | ||
| g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) | ||
| } else { | ||
| g.Expect(pod.Status).Should(Equal("Running"), pod.Name) | ||
| } | ||
| } | ||
| }, "620s", "5s").Should(Succeed()) | ||
| _, _ = e2e.ParsePods(kubeConfigFile, true) | ||
| }) | ||
| It("Checks that btrfs snapshots exist", func() { | ||
| cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs" | ||
| res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| Expect(res).To(ContainSubstring("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/2")) | ||
| Expect(res).To(ContainSubstring("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/3")) | ||
| }) | ||
| }) | ||
| }) | ||
|
|
||
| var failed bool | ||
| var _ = AfterEach(func() { | ||
| failed = failed || CurrentSpecReport().Failed() | ||
| }) | ||
|
|
||
| var _ = AfterSuite(func() { | ||
| if failed && !*ci { | ||
| fmt.Println("FAILED!") | ||
| } else { | ||
| Expect(e2e.DestroyCluster()).To(Succeed()) | ||
| Expect(os.Remove(kubeConfigFile)).To(Succeed()) | ||
| } | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.