From 6ef013d7c0f758935df298e8120958daa19a9e9a Mon Sep 17 00:00:00 2001 From: danfengl Date: Mon, 21 Aug 2023 06:42:34 +0000 Subject: [PATCH] Replace pod with deployment for E2E test Signed-off-by: danfengl --- test/e2e/basic/pvc-selected-node-changing.go | 4 +- test/e2e/basic/storage-class-changing.go | 79 ++++++++++------ test/e2e/bsl-mgmt/deletion.go | 4 +- test/e2e/pv-backup/pv-backup-filter.go | 6 +- test/e2e/resource-filtering/base.go | 2 +- test/e2e/resource-filtering/exclude_label.go | 2 +- test/e2e/resource-filtering/label_selector.go | 2 +- test/e2e/schedule/ordered_resources.go | 2 +- test/e2e/test/test.go | 4 +- test/e2e/upgrade/upgrade.go | 1 + test/e2e/util/csi/common.go | 58 +++++++++++- test/e2e/util/k8s/common.go | 92 +++++++++++++------ test/e2e/util/k8s/configmap.go | 24 ++++- test/e2e/util/k8s/deployment.go | 92 ++++++++++++++----- test/e2e/util/k8s/sc.go | 10 ++ test/e2e/util/velero/velero_utils.go | 36 ++++---- 16 files changed, 302 insertions(+), 116 deletions(-) diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index 9b31966a32..b0f0973625 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -98,7 +98,7 @@ func (p *PVCSelectedNodeChanging) CreateResources() error { By("Prepare ConfigMap data", func() { nodeNameList, err := GetWorkerNodes(context.Background()) Expect(err).To(Succeed()) - Expect(len(nodeNameList) > 2).To(Equal(true)) + Expect(len(nodeNameList) > 1).To(Equal(true)) for _, nodeName := range nodeNameList { if nodeName != p.oldNodeName { p.newNodeName = nodeName @@ -142,7 +142,7 @@ func (p *PVCSelectedNodeChanging) Restore() error { } func (p *PVCSelectedNodeChanging) Verify() error { By(fmt.Sprintf("PVC selected node should be %s", p.newNodeName), func() { - pvcNameList, err := GetPvcByPodName(context.Background(), p.mappedNS, p.pvcName) + pvcNameList, err := GetPvcByPVCName(context.Background(), p.mappedNS, p.pvcName) Expect(err).To(Succeed()) Expect(len(pvcNameList)).Should(Equal(1)) pvc, err := GetPVC(context.Background(), p.Client, p.mappedNS, pvcNameList[0]) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index 4ae1d71c68..ef1b2fde9e 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -24,44 +25,44 @@ type StorageClasssChanging struct { namespace string srcStorageClass string desStorageClass string + pvcName string volume string podName string mappedNS string + deploymentName string + CaseBaseName string } const SCCBaseName string = "scc-" -var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{ - namespace: SCCBaseName + "1", TestCase: TestCase{NSBaseName: SCCBaseName}}) +var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{}) func (s *StorageClasssChanging) Init() error { + s.TestCase.Init() + UUIDgen, err := uuid.NewRandom() + Expect(err).To(Succeed()) + s.CaseBaseName = SCCBaseName + UUIDgen.String() + s.namespace = SCCBaseName + UUIDgen.String() + s.BackupName = "backup-" + s.CaseBaseName + s.RestoreName = "restore-" + s.CaseBaseName + s.mappedNS = s.namespace + "-mapped" s.VeleroCfg = VeleroCfg s.Client = *s.VeleroCfg.ClientToInstallVelero - s.NSBaseName = SCCBaseName - s.namespace = s.NSBaseName + UUIDgen.String() - s.mappedNS = s.namespace + "-mapped" s.TestMsg = &TestMSG{ Desc: "Changing PV/PVC Storage Classes", FailedMSG: "Failed to changing PV/PVC Storage Classes", Text: "Change the storage class of persistent volumes and persistent" + " volume claims during restores", } - s.BackupName = "backup-sc-" + UUIDgen.String() - s.RestoreName = "restore-" + UUIDgen.String() s.srcStorageClass = "default" - s.desStorageClass = "e2e-storage-class" + s.desStorageClass = StorageClassName s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction", "velero.io/plugin-config": ""} s.data = map[string]string{s.srcStorageClass: s.desStorageClass} s.configmaptName = "change-storage-class-config" s.volume = "volume-1" + s.pvcName = fmt.Sprintf("pvc-%s", s.volume) s.podName = "pod-1" - return nil -} - -func (s *StorageClasssChanging) StartRun() error { - s.BackupName = s.BackupName + "backup-" + UUIDgen.String() - s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String() s.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", s.BackupName, "--include-namespaces", s.namespace, @@ -74,9 +75,12 @@ func (s *StorageClasssChanging) StartRun() error { return nil } func (s *StorageClasssChanging) CreateResources() error { - s.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + label := map[string]string{ + "app": "test", + } + s.Ctx, _ = context.WithTimeout(context.Background(), 10*time.Minute) By(fmt.Sprintf("Create a storage class %s", s.desStorageClass), func() { - Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", + Expect(InstallStorageClass(s.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed()) }) By(fmt.Sprintf("Create namespace %s", s.namespace), func() { @@ -84,10 +88,20 @@ func (s *StorageClasssChanging) CreateResources() error { fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) - By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() { - _, err := CreatePod(s.Client, s.namespace, s.podName, s.srcStorageClass, "", []string{s.volume}, nil, nil) + By(fmt.Sprintf("Create a deployment in namespace %s", s.VeleroCfg.VeleroNamespace), func() { + + pvc, err := CreatePVC(s.Client, s.namespace, s.pvcName, s.srcStorageClass, nil) + Expect(err).To(Succeed()) + vols := CreateVolumes(pvc.Name, []string{s.volume}) + + deployment := NewDeployment(s.CaseBaseName, s.namespace, 1, label, nil).WithVolume(vols).Result() + deployment, err = CreateDeployment(s.Client.ClientGo, s.namespace, deployment) + Expect(err).To(Succeed()) + s.deploymentName = deployment.Name + err = WaitForReadyDeployment(s.Client.ClientGo, s.namespace, s.deploymentName) Expect(err).To(Succeed()) }) + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", s.configmaptName, s.VeleroCfg.VeleroNamespace), func() { _, err := CreateConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName, s.labels, s.data) Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", s.VeleroCfg.VeleroNamespace)) @@ -97,19 +111,14 @@ func (s *StorageClasssChanging) CreateResources() error { func (s *StorageClasssChanging) Destroy() error { By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.srcStorageClass), func() { - pvName, err := GetPVByPodName(s.Client, s.namespace, s.volume) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName)) + pvName, err := GetPVByPVCName(s.Client, s.namespace, s.pvcName) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by PVC name %s", s.pvcName)) pv, err := GetPersistentVolume(s.Ctx, s.Client, s.namespace, pvName) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName)) - fmt.Println(pv) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by name %s", pvName)) Expect(pv.Spec.StorageClassName).To(Equal(s.srcStorageClass), fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.srcStorageClass)) }) - By(fmt.Sprintf("Start to destroy namespace %s......", s.NSBaseName), func() { - Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.NSBaseName)).To(Succeed(), - fmt.Sprintf("Failed to delete namespace %s", s.NSBaseName)) - }) return nil } @@ -129,14 +138,26 @@ func (s *StorageClasssChanging) Restore() error { } func (s *StorageClasssChanging) Verify() error { By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.desStorageClass), func() { - time.Sleep(1 * time.Minute) - pvName, err := GetPVByPodName(s.Client, s.mappedNS, s.volume) + Expect(WaitForReadyDeployment(s.Client.ClientGo, s.mappedNS, s.deploymentName)).To(Succeed()) + pvName, err := GetPVByPVCName(s.Client, s.mappedNS, s.pvcName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName)) pv, err := GetPersistentVolume(s.Ctx, s.Client, s.mappedNS, pvName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName)) - fmt.Println(pv) Expect(pv.Spec.StorageClassName).To(Equal(s.desStorageClass), fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.desStorageClass)) }) return nil } + +func (s *StorageClasssChanging) Clean() error { + if !s.VeleroCfg.Debug { + By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() { + Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) + }) + DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName) + DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass) + s.TestCase.Clean() + } + return nil +} diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 5d8619a8c7..b4d0b256c8 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -169,12 +169,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) By("Get all 2 PVCs of Kibishii and label them seprately ", func() { - pvc, err := GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_1) + pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(len(pvc)).To(Equal(1)) pvc1 := pvc[0] - pvc, err = GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_2) + pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(len(pvc)).To(Equal(1)) diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 47785309ca..5feadaef7e 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -124,7 +124,7 @@ func (p *PVBackupFiltering) CreateResources() error { WaitForPods(p.Ctx, p.Client, ns, p.podsList[index]) for i, pod := range p.podsList[index] { for j := range p.volumesList[i] { - Expect(CreateFileToPod(p.Ctx, ns, pod, p.volumesList[i][j], + Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j], FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) } } @@ -182,7 +182,7 @@ func fileContent(namespace, podName, volume string) string { } func fileExist(ctx context.Context, namespace, podName, volume string) error { - c, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME) + c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) if err != nil { return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ", FILE_NAME, volume, podName, namespace)) @@ -197,7 +197,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error { } } func fileNotExist(ctx context.Context, namespace, podName, volume string) error { - _, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME) + _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) if err != nil { return nil } else { diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index b13bf87023..35f5e3f77f 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -84,7 +84,7 @@ func (f *FilteringCase) CreateResources() error { } //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil) + deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result() deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index f04308ea1c..7644a1c86e 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error { } //Create deployment: to be included fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil) + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result() deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 2641f60b80..2053daed35 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error { //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil) + deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result() deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 166913c417..f220d9d441 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -159,7 +159,7 @@ func (o *OrderedResources) CreateResources() error { //Create deployment deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName) fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) - deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil) + deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result() deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err)) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index b0b4d7ac8a..7a6da70bdc 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -34,6 +34,8 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) +const StorageClassName = "e2e-storage-class" + /* The VeleroBackupRestoreTest interface is just could be suit for the cases that follow the test flow of create resources, backup, delete test resource, restore and verify. @@ -83,7 +85,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() { flag.Parse() veleroCfg := test.GetTestCase().VeleroCfg // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") { + if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } if veleroCfg.InstallVelero { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 0bf0ccc966..5238e259ff 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -120,6 +120,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.RestoreHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" tmpCfgForOldVeleroInstall.UploaderType = "" + tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots if supportUploaderType { tmpCfgForOldVeleroInstall.UseRestic = false tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go index 51fb5bc2ce..cbd397f385 100644 --- a/test/e2e/util/csi/common.go +++ b/test/e2e/util/csi/common.go @@ -93,7 +93,7 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error return snapshotHandleList, nil } func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) { - pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) + pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName) if err != nil { return "", err } @@ -129,13 +129,61 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { var err error var snapshotContentNameList []string - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || - len(snapshotContentNameList) != expectedCount { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + if apiVersion == "v1beta1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else if apiVersion == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } + } else { + return nil, errors.New("API version is invalid") + } + if len(snapshotContentNameList) != expectedCount { + return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) } fmt.Println(snapshotContentNameList) return snapshotContentNameList, nil } + +func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { + _, snapshotClient, err := GetClients() + if err != nil { + return nil, err + } + vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err1 != nil { + return nil, err + } + + var snapshotHandleList []string + for _, i := range vscList.Items { + if i.Status == nil { + fmt.Println("SnapshotHandle Status s nil") + continue + } + if i.Status.SnapshotHandle == nil { + fmt.Println("SnapshotHandle is nil") + continue + } + + if i.Labels == nil { + fmt.Println("VolumeSnapshotContents label is nil") + continue + } + + if i.Labels["velero.io/backup-name"] == backupName { + tmp := strings.Split(*i.Status.SnapshotHandle, "/") + snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) + } + } + + if len(snapshotHandleList) == 0 { + fmt.Printf("No VolumeSnapshotContent from backup %s", backupName) + } + return snapshotHandleList, nil +} diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index eb0ec43c67..6c45cb723e 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -33,7 +33,7 @@ import ( "github.com/vmware-tanzu/velero/test/e2e/util/common" ) -// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. +// ensureClusterExists returns whether or not a Kubernetes cluster exists for tests to be run on. func EnsureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } @@ -81,7 +81,7 @@ func WaitForPods(ctx context.Context, client TestClient, namespace string, pods return nil } -func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, error) { +func GetPvcByPVCName(ctx context.Context, namespace, pvcName string) ([]string, error) { // Example: // NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE // kibishii-data-kibishii-deployment-0 Bound pvc-94b9fdf2-c30f-4a7b-87bf-06eadca0d5b6 1Gi RWO kibishii-storage-class 115s @@ -94,7 +94,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, cmd = &common.OsCommandLine{ Cmd: "grep", - Args: []string{podName}, + Args: []string{pvcName}, } cmds = append(cmds, cmd) @@ -200,6 +200,7 @@ func AddLabelToCRD(ctx context.Context, crd, label string) error { func KubectlApplyByFile(ctx context.Context, file string) error { args := []string{"apply", "-f", file, "--force=true"} + fmt.Println(args) return exec.CommandContext(ctx, "kubectl", args...).Run() } @@ -213,39 +214,20 @@ func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error { return err } -func GetAPIVersions(client *TestClient, name string) ([]string, error) { - var version []string - APIGroup, err := client.ClientGo.Discovery().ServerGroups() - if err != nil { - return nil, errors.Wrap(err, "Fail to get server API groups") - } - for _, group := range APIGroup.Groups { - fmt.Println(group.Name) - if group.Name == name { - for _, v := range group.Versions { - fmt.Println(v.Version) - version = append(version, v.Version) - } - return version, nil - } - } - return nil, errors.New("Server API groups is empty") -} - -func GetPVByPodName(client TestClient, namespace, podName string) (string, error) { - pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) +func GetPVByPVCName(client TestClient, namespace, pvcName string) (string, error) { + pvcList, err := GetPvcByPVCName(context.Background(), namespace, pvcName) if err != nil { return "", err } if len(pvcList) != 1 { - return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace)) + return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s but got %v", pvcName, namespace, pvcList)) } pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0]) if err != nil { return "", err } if len(pvList) != 1 { - return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace)) + return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], pvcName, namespace)) } pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0]) fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"]) @@ -255,15 +237,30 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error return pv_value.Name, nil } -func CreateFileToPod(ctx context.Context, namespace, podName, volume, filename, content string) error { - arg := []string{"exec", "-n", namespace, "-c", podName, podName, +func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) { + for i, volume := range volumeNameList { + vols = append(vols, &corev1.Volume{ + Name: volume, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: fmt.Sprintf("pvc-%d", i), + ReadOnly: false, + }, + }, + }) + } + return +} + +func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)} cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) return cmd.Run() } -func ReadFileFromPodVolume(ctx context.Context, namespace, podName, volume, filename string) (string, error) { - arg := []string{"exec", "-n", namespace, "-c", podName, podName, +func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, error) { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "cat", fmt.Sprintf("/%s/%s", volume, filename)} cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) @@ -331,3 +328,38 @@ func GetAllService(ctx context.Context) (string, error) { } return stdout, nil } + +func CreateVolumes(pvcName string, volumeNameList []string) (vols []*corev1.Volume) { + vols = []*corev1.Volume{} + for _, volume := range volumeNameList { + vols = append(vols, &corev1.Volume{ + Name: volume, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + ReadOnly: false, + }, + }, + }) + } + return +} + +func GetAPIVersions(client *TestClient, name string) ([]string, error) { + var version []string + APIGroup, err := client.ClientGo.Discovery().ServerGroups() + if err != nil { + return nil, errors.Wrap(err, "Fail to get server API groups") + } + for _, group := range APIGroup.Groups { + fmt.Println(group.Name) + if group.Name == name { + for _, v := range group.Versions { + fmt.Println(v.Version) + version = append(version, v.Version) + } + return version, nil + } + } + return nil, errors.New("Server API groups is empty") +} diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go index 0f311fa1d2..e4edd0667b 100644 --- a/test/e2e/util/k8s/configmap.go +++ b/test/e2e/util/k8s/configmap.go @@ -42,6 +42,20 @@ func CreateConfigMap(c clientset.Interface, ns, name string, labels, data map[st return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{}) } +func CreateConfigMapFromYAMLData(c clientset.Interface, yamlData, cmName, namespace string) error { + cmData := make(map[string]string) + cmData[cmName] = yamlData + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: namespace, + }, + Data: cmData, + } + _, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), cm, metav1.CreateOptions{}) + return err +} + // WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns. func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error { return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { @@ -57,10 +71,18 @@ func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) } -func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { +func DeleteConfigmap(c clientset.Interface, ns, name string) error { if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns)) } + return nil +} + +func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { + if err := DeleteConfigmap(c, ns, name); err != nil { + return err + } + return waitutil.PollImmediateInfinite(5*time.Second, func() (bool, error) { if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go index e9f5572fbf..15a5d372f6 100644 --- a/test/e2e/util/k8s/deployment.go +++ b/test/e2e/util/k8s/deployment.go @@ -26,6 +26,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + + "github.com/vmware-tanzu/velero/pkg/util/boolptr" ) const ( @@ -35,46 +37,90 @@ const ( PollTimeout = 15 * time.Minute ) +// DeploymentBuilder builds Deployment objects. +type DeploymentBuilder struct { + *apps.Deployment +} + +func (d *DeploymentBuilder) Result() *apps.Deployment { + return d.Deployment +} + // newDeployment returns a RollingUpdate Deployment with a fake container image -func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment { +func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *DeploymentBuilder { if containers == nil { containers = []v1.Container{ { - Name: fmt.Sprintf("container-%s", "busybox"), + Name: "container-busybox", Image: "gcr.io/velero-gcp/busybox:latest", Command: []string{"sleep", "1000000"}, + // Make pod obeys the restricted pod security standards. + SecurityContext: &v1.SecurityContext{ + AllowPrivilegeEscalation: boolptr.False(), + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, + RunAsNonRoot: boolptr.True(), + RunAsUser: func(i int64) *int64 { return &i }(65534), + RunAsGroup: func(i int64) *int64 { return &i }(65534), + SeccompProfile: &v1.SeccompProfile{ + Type: v1.SeccompProfileTypeRuntimeDefault, + }, + }, }, } } - return &apps.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: "apps/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - Labels: labels, - }, - Spec: apps.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Strategy: apps.DeploymentStrategy{ - Type: apps.RollingUpdateDeploymentStrategyType, - RollingUpdate: new(apps.RollingUpdateDeployment), + return &DeploymentBuilder{ + &apps.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: labels, }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + RollingUpdate: new(apps.RollingUpdateDeployment), }, - Spec: v1.PodSpec{ - Containers: containers, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + SecurityContext: &v1.PodSecurityContext{ + FSGroup: func(i int64) *int64 { return &i }(65534), + FSGroupChangePolicy: func(policy v1.PodFSGroupChangePolicy) *v1.PodFSGroupChangePolicy { return &policy }(v1.FSGroupChangeAlways), + }, + Containers: containers, + }, }, }, }, } } +func (d *DeploymentBuilder) WithVolume(volumes []*v1.Volume) *DeploymentBuilder { + vmList := []v1.VolumeMount{} + for _, v := range volumes { + vmList = append(vmList, v1.VolumeMount{ + Name: v.Name, + MountPath: "/" + v.Name, + }) + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, *v) + + } + + // NOTE here just mount volumes to the first container + d.Spec.Template.Spec.Containers[0].VolumeMounts = vmList + return d +} + func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error { _, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) return err diff --git a/test/e2e/util/k8s/sc.go b/test/e2e/util/k8s/sc.go index bcc5cbfed6..3878018b2f 100644 --- a/test/e2e/util/k8s/sc.go +++ b/test/e2e/util/k8s/sc.go @@ -3,6 +3,9 @@ package k8s import ( "context" "fmt" + + "github.com/pkg/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func InstallStorageClass(ctx context.Context, yaml string) error { @@ -10,3 +13,10 @@ func InstallStorageClass(ctx context.Context, yaml string) error { err := KubectlApplyByFile(ctx, yaml) return err } + +func DeleteStorageClass(ctx context.Context, client TestClient, name string) error { + if err := client.ClientGo.StorageV1().StorageClasses().Delete(ctx, name, v1.DeleteOptions{}); err != nil { + return errors.Wrapf(err, "Could not retrieve storage classes %s", name) + } + return nil +} diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index b8d57c6213..d7c33bce74 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -1056,15 +1056,25 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace return common.GetListByCmdPipes(ctx, cmds) } -func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) { +func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { var snapshotCheckPoint SnapshotCheckPoint - var err error + snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp - snapshotCheckPoint.PodName = kibishiiPodNameList + snapshotCheckPoint.PodName = KibishiiPVCNameList if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { + resourceName := "snapshot.storage.k8s.io" + + srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName) + + if err != nil { + return snapshotCheckPoint, err + } + if len(srcVersions) == 0 { + return snapshotCheckPoint, errors.New("Fail to get APIVersion") + } + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } } @@ -1183,12 +1193,9 @@ func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]stri Args: []string{"get", "deploy", "-n", veleroCfg.VeleroNamespace, "-ojson"}, } cmds = append(cmds, cmd) - var args string - if veleroCfg.CloudProvider == "vsphere" { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) - } else { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) - } + + args := fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) + cmd = &common.OsCommandLine{ Cmd: "sed", Args: []string{args}, @@ -1236,12 +1243,9 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string) Args: []string{dsjson}, } cmds = append(cmds, cmd) - var args string - if veleroCfg.CloudProvider == "vsphere" { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) - } else { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) - } + + args := fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) + cmd = &common.OsCommandLine{ Cmd: "sed", Args: []string{args},