Merge pull request #6684 from danfengliu/fix-storage-class-failure-issue

Replace pod with deployment for E2E test

test/e2e/basic/pvc-selected-node-changing.go

@@ -98,7 +98,7 @@ func (p *PVCSelectedNodeChanging) CreateResources() error {
 	By("Prepare ConfigMap data", func() {
 		nodeNameList, err := GetWorkerNodes(context.Background())
 		Expect(err).To(Succeed())
-		Expect(len(nodeNameList) > 2).To(Equal(true))
+		Expect(len(nodeNameList) > 1).To(Equal(true))
 		for _, nodeName := range nodeNameList {
 			if nodeName != p.oldNodeName {
 				p.newNodeName = nodeName
@@ -142,7 +142,7 @@ func (p *PVCSelectedNodeChanging) Restore() error {
 }
 func (p *PVCSelectedNodeChanging) Verify() error {
 	By(fmt.Sprintf("PVC selected node should be %s", p.newNodeName), func() {
-		pvcNameList, err := GetPvcByPodName(context.Background(), p.mappedNS, p.pvcName)
+		pvcNameList, err := GetPvcByPVCName(context.Background(), p.mappedNS, p.pvcName)
 		Expect(err).To(Succeed())
 		Expect(len(pvcNameList)).Should(Equal(1))
 		pvc, err := GetPVC(context.Background(), p.Client, p.mappedNS, pvcNameList[0])

test/e2e/basic/storage-class-changing.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -24,44 +25,44 @@ type StorageClasssChanging struct {
 	namespace       string
 	srcStorageClass string
 	desStorageClass string
+	pvcName         string
 	volume          string
 	podName         string
 	mappedNS        string
+	deploymentName  string
+	CaseBaseName    string
 }
 
 const SCCBaseName string = "scc-"
 
-var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{
-	namespace: SCCBaseName + "1", TestCase: TestCase{NSBaseName: SCCBaseName}})
+var StorageClasssChangingTest func() = TestFunc(&StorageClasssChanging{})
 
 func (s *StorageClasssChanging) Init() error {
+	s.TestCase.Init()
+	UUIDgen, err := uuid.NewRandom()
+	Expect(err).To(Succeed())
+	s.CaseBaseName = SCCBaseName + UUIDgen.String()
+	s.namespace = SCCBaseName + UUIDgen.String()
+	s.BackupName = "backup-" + s.CaseBaseName
+	s.RestoreName = "restore-" + s.CaseBaseName
+	s.mappedNS = s.namespace + "-mapped"
 	s.VeleroCfg = VeleroCfg
 	s.Client = *s.VeleroCfg.ClientToInstallVelero
-	s.NSBaseName = SCCBaseName
-	s.namespace = s.NSBaseName + UUIDgen.String()
-	s.mappedNS = s.namespace + "-mapped"
 	s.TestMsg = &TestMSG{
 		Desc:      "Changing PV/PVC Storage Classes",
 		FailedMSG: "Failed to changing PV/PVC Storage Classes",
 		Text: "Change the storage class of persistent volumes and persistent" +
 			" volume claims during restores",
 	}
-	s.BackupName = "backup-sc-" + UUIDgen.String()
-	s.RestoreName = "restore-" + UUIDgen.String()
 	s.srcStorageClass = "default"
-	s.desStorageClass = "e2e-storage-class"
+	s.desStorageClass = StorageClassName
 	s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
 		"velero.io/plugin-config": ""}
 	s.data = map[string]string{s.srcStorageClass: s.desStorageClass}
 	s.configmaptName = "change-storage-class-config"
 	s.volume = "volume-1"
+	s.pvcName = fmt.Sprintf("pvc-%s", s.volume)
 	s.podName = "pod-1"
-	return nil
-}
-
-func (s *StorageClasssChanging) StartRun() error {
-	s.BackupName = s.BackupName + "backup-" + UUIDgen.String()
-	s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String()
 	s.BackupArgs = []string{
 		"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", s.BackupName,
 		"--include-namespaces", s.namespace,
@@ -74,20 +75,33 @@ func (s *StorageClasssChanging) StartRun() error {
 	return nil
 }
 func (s *StorageClasssChanging) CreateResources() error {
-	s.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
+	label := map[string]string{
+		"app": "test",
+	}
+	s.Ctx, _ = context.WithTimeout(context.Background(), 10*time.Minute)
 	By(fmt.Sprintf("Create a storage class %s", s.desStorageClass), func() {
-		Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml",
+		Expect(InstallStorageClass(s.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml",
 			s.VeleroCfg.CloudProvider))).To(Succeed())
 	})
 	By(fmt.Sprintf("Create namespace %s", s.namespace), func() {
 		Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(),
 			fmt.Sprintf("Failed to create namespace %s", s.namespace))
 	})
 
-	By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() {
-		_, err := CreatePod(s.Client, s.namespace, s.podName, s.srcStorageClass, "", []string{s.volume}, nil, nil)
+	By(fmt.Sprintf("Create a deployment in namespace %s", s.VeleroCfg.VeleroNamespace), func() {
+
+		pvc, err := CreatePVC(s.Client, s.namespace, s.pvcName, s.srcStorageClass, nil)
+		Expect(err).To(Succeed())
+		vols := CreateVolumes(pvc.Name, []string{s.volume})
+
+		deployment := NewDeployment(s.CaseBaseName, s.namespace, 1, label, nil).WithVolume(vols).Result()
+		deployment, err = CreateDeployment(s.Client.ClientGo, s.namespace, deployment)
+		Expect(err).To(Succeed())
+		s.deploymentName = deployment.Name
+		err = WaitForReadyDeployment(s.Client.ClientGo, s.namespace, s.deploymentName)
 		Expect(err).To(Succeed())
 	})
+
 	By(fmt.Sprintf("Create ConfigMap %s in namespace %s", s.configmaptName, s.VeleroCfg.VeleroNamespace), func() {
 		_, err := CreateConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName, s.labels, s.data)
 		Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", s.VeleroCfg.VeleroNamespace))
@@ -97,19 +111,14 @@ func (s *StorageClasssChanging) CreateResources() error {
 
 func (s *StorageClasssChanging) Destroy() error {
 	By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.srcStorageClass), func() {
-		pvName, err := GetPVByPodName(s.Client, s.namespace, s.volume)
-		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName))
+		pvName, err := GetPVByPVCName(s.Client, s.namespace, s.pvcName)
+		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by PVC name %s", s.pvcName))
 		pv, err := GetPersistentVolume(s.Ctx, s.Client, s.namespace, pvName)
-		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName))
-		fmt.Println(pv)
+		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by name %s", pvName))
 		Expect(pv.Spec.StorageClassName).To(Equal(s.srcStorageClass),
 			fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.srcStorageClass))
 	})
 
-	By(fmt.Sprintf("Start to destroy namespace %s......", s.NSBaseName), func() {
-		Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.NSBaseName)).To(Succeed(),
-			fmt.Sprintf("Failed to delete namespace %s", s.NSBaseName))
-	})
 	return nil
 }
 
@@ -129,14 +138,26 @@ func (s *StorageClasssChanging) Restore() error {
 }
 func (s *StorageClasssChanging) Verify() error {
 	By(fmt.Sprintf("Expect storage class of PV %s to be %s ", s.volume, s.desStorageClass), func() {
-		time.Sleep(1 * time.Minute)
-		pvName, err := GetPVByPodName(s.Client, s.mappedNS, s.volume)
+		Expect(WaitForReadyDeployment(s.Client.ClientGo, s.mappedNS, s.deploymentName)).To(Succeed())
+		pvName, err := GetPVByPVCName(s.Client, s.mappedNS, s.pvcName)
 		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV name by pod name %s", s.podName))
 		pv, err := GetPersistentVolume(s.Ctx, s.Client, s.mappedNS, pvName)
 		Expect(err).To(Succeed(), fmt.Sprintf("Failed to get PV by pod name %s", s.podName))
-		fmt.Println(pv)
 		Expect(pv.Spec.StorageClassName).To(Equal(s.desStorageClass),
 			fmt.Sprintf("PV storage %s is not as expected %s", pv.Spec.StorageClassName, s.desStorageClass))
 	})
 	return nil
 }
+
+func (s *StorageClasssChanging) Clean() error {
+	if !s.VeleroCfg.Debug {
+		By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() {
+			Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(),
+				fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName))
+		})
+		DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName)
+		DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass)
+		s.TestCase.Clean()
+	}
+	return nil
+}

test/e2e/bsl-mgmt/deletion.go

@@ -169,12 +169,12 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			})
 
 			By("Get all 2 PVCs of Kibishii and label them seprately ", func() {
-				pvc, err := GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_1)
+				pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1)
 				Expect(err).To(Succeed())
 				fmt.Println(pvc)
 				Expect(len(pvc)).To(Equal(1))
 				pvc1 := pvc[0]
-				pvc, err = GetPvcByPodName(context.Background(), bslDeletionTestNs, podName_2)
+				pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2)
 				Expect(err).To(Succeed())
 				fmt.Println(pvc)
 				Expect(len(pvc)).To(Equal(1))

test/e2e/pv-backup/pv-backup-filter.go

@@ -124,7 +124,7 @@ func (p *PVBackupFiltering) CreateResources() error {
 				WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])
 				for i, pod := range p.podsList[index] {
 					for j := range p.volumesList[i] {
-						Expect(CreateFileToPod(p.Ctx, ns, pod, p.volumesList[i][j],
+						Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j],
 							FILE_NAME, fileContent(ns, pod, p.volumesList[i][j]))).To(Succeed())
 					}
 				}
@@ -182,7 +182,7 @@ func fileContent(namespace, podName, volume string) string {
 }
 
 func fileExist(ctx context.Context, namespace, podName, volume string) error {
-	c, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
+	c, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ",
 			FILE_NAME, volume, podName, namespace))
@@ -197,7 +197,7 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error {
 	}
 }
 func fileNotExist(ctx context.Context, namespace, podName, volume string) error {
-	_, err := ReadFileFromPodVolume(ctx, namespace, podName, volume, FILE_NAME)
+	_, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME)
 	if err != nil {
 		return nil
 	} else {

test/e2e/resource-filtering/base.go

@@ -84,7 +84,7 @@ func (f *FilteringCase) CreateResources() error {
 		}
 		//Create deployment
 		fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
-		deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil)
+		deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil).Result()
 		deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/resource-filtering/exclude_label.go

@@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error {
 	}
 	//Create deployment: to be included
 	fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
-	deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil)
+	deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil).Result()
 	deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/resource-filtering/label_selector.go

@@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error {
 		//Create deployment
 		fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
 
-		deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil)
+		deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil).Result()
 		deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

test/e2e/schedule/ordered_resources.go

@@ -159,7 +159,7 @@ func (o *OrderedResources) CreateResources() error {
 	//Create deployment
 	deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
 	fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
-	deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil)
+	deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil).Result()
 	deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))

test/e2e/test/test.go

@@ -34,6 +34,8 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
+const StorageClassName = "e2e-storage-class"
+
 /*
 The VeleroBackupRestoreTest interface is just could be suit for the cases that follow the test flow of
 create resources, backup, delete test resource, restore and verify.
@@ -83,7 +85,7 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 			flag.Parse()
 			veleroCfg := test.GetTestCase().VeleroCfg
 			// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
-			if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") {
+			if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") {
 				Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
 			}
 			if veleroCfg.InstallVelero {

test/e2e/upgrade/upgrade.go

@@ -120,6 +120,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				tmpCfgForOldVeleroInstall.RestoreHelperImage = ""
 				tmpCfgForOldVeleroInstall.Plugins = ""
 				tmpCfgForOldVeleroInstall.UploaderType = ""
+				tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots
 				if supportUploaderType {
 					tmpCfgForOldVeleroInstall.UseRestic = false
 					tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots

test/e2e/util/csi/common.go

@@ -93,7 +93,7 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	return snapshotHandleList, nil
 }
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
-	pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
+	pvcList, err := GetPvcByPVCName(context.Background(), namespace, podName)
 	if err != nil {
 		return "", err
 	}
@@ -129,13 +129,61 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
 }
 
-func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
+func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) {
 	var err error
 	var snapshotContentNameList []string
-	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil ||
-		len(snapshotContentNameList) != expectedCount {
-		return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+	if apiVersion == "v1beta1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else if apiVersion == "v1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else {
+		return nil, errors.New("API version is invalid")
+	}
+	if len(snapshotContentNameList) != expectedCount {
+		return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))
 	}
 	fmt.Println(snapshotContentNameList)
 	return snapshotContentNameList, nil
 }
+
+func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) {
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return nil, err
+	}
+	vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
+	if err1 != nil {
+		return nil, err
+	}
+
+	var snapshotHandleList []string
+	for _, i := range vscList.Items {
+		if i.Status == nil {
+			fmt.Println("SnapshotHandle Status s nil")
+			continue
+		}
+		if i.Status.SnapshotHandle == nil {
+			fmt.Println("SnapshotHandle is nil")
+			continue
+		}
+
+		if i.Labels == nil {
+			fmt.Println("VolumeSnapshotContents label is nil")
+			continue
+		}
+
+		if i.Labels["velero.io/backup-name"] == backupName {
+			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
+			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
+		}
+	}
+
+	if len(snapshotHandleList) == 0 {
+		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+	}
+	return snapshotHandleList, nil
+}