diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 57d4998237..e61ba70bbd 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -70,13 +70,10 @@ jobs: focus: # tests to focus on, use `|` to concatenate multiple regexes to run on the same job # ordered according to e2e_suite_test.go order - - Basic\]\[ClusterResource - - ResourceFiltering - - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR - - Schedule\]\[OrderedResources - - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic - - Basic\]\[Nodeport - - Basic\]\[StorageClass + - Basic && (ClusterResource || NodePort || StorageClass) + - ResourceFiltering && !Restic + - ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources + - (NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic) fail-fast: false steps: - name: Set up Go @@ -128,13 +125,18 @@ jobs: curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - GOPATH=~/go CLOUD_PROVIDER=kind \ - OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ - CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ - ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ - ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ - GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \ - GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \ + GOPATH=~/go \ + CLOUD_PROVIDER=kind \ + OBJECT_STORE_PROVIDER=aws \ + BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + CREDS_FILE=/tmp/credential \ + BSL_BUCKET=bucket \ + ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ + ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + ADDITIONAL_CREDS_FILE=/tmp/credential \ + ADDITIONAL_BSL_BUCKET=additional-bucket \ + VELERO_IMAGE=velero:pr-test \ + GINKGO_LABELS="${{ matrix.focus }}" \ make -C test/ run-e2e timeout-minutes: 30 - name: Upload debug bundle diff --git a/.github/workflows/pr-codespell.yml b/.github/workflows/pr-codespell.yml index ca733318ca..0d3138e403 100644 --- a/.github/workflows/pr-codespell.yml +++ b/.github/workflows/pr-codespell.yml @@ -15,7 +15,7 @@ jobs: with: # ignore the config/.../crd.go file as it's generated binary data that is edited elswhere. skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE - ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme + ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin check_filenames: true check_hidden: true diff --git a/test/Makefile b/test/Makefile index fbc0e08e35..bdf87183b3 100644 --- a/test/Makefile +++ b/test/Makefile @@ -47,10 +47,9 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) GINKGO := $(GOBIN)/ginkgo KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin -GINKGO_FOCUS ?= -GINKGO_SKIP ?= -SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)") -FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)") +# Please reference to this document for Ginkgo label spec format. +# https://onsi.github.io/ginkgo/#spec-labels +GINKGO_LABELS ?= VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main PLUGINS ?= @@ -129,26 +128,26 @@ VELERO_POD_CPU_REQUEST ?= 2 VELERO_POD_MEM_REQUEST ?= 2Gi POD_VOLUME_OPERATION_TIMEOUT ?= 6h -COMMON_ARGS := -velerocli=$(VELERO_CLI) \ - -velero-image=$(VELERO_IMAGE) \ - -plugins=$(PLUGINS) \ - -velero-version=$(VERSION) \ - -restore-helper-image=$(RESTORE_HELPER_IMAGE) \ - -velero-namespace=$(VELERO_NAMESPACE) \ - -credentials-file=$(CREDS_FILE) \ - -bucket=$(BSL_BUCKET) \ - -prefix=$(BSL_PREFIX) \ - -bsl-config=$(BSL_CONFIG) \ - -vsl-config=$(VSL_CONFIG) \ - -cloud-provider=$(CLOUD_PROVIDER) \ - -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ - -features=$(FEATURES) \ - -install-velero=$(INSTALL_VELERO) \ - -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ - -debug-e2e-test=$(DEBUG_E2E_TEST) \ - -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ - -uploader-type=$(UPLOADER_TYPE) \ - -debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) +COMMON_ARGS := --velerocli=$(VELERO_CLI) \ + --velero-image=$(VELERO_IMAGE) \ + --plugins=$(PLUGINS) \ + --velero-version=$(VERSION) \ + --restore-helper-image=$(RESTORE_HELPER_IMAGE) \ + --velero-namespace=$(VELERO_NAMESPACE) \ + --credentials-file=$(CREDS_FILE) \ + --bucket=$(BSL_BUCKET) \ + --prefix=$(BSL_PREFIX) \ + --bsl-config=$(BSL_CONFIG) \ + --vsl-config=$(VSL_CONFIG) \ + --cloud-provider=$(CLOUD_PROVIDER) \ + --object-store-provider="$(OBJECT_STORE_PROVIDER)" \ + --features=$(FEATURES) \ + --install-velero=$(INSTALL_VELERO) \ + --registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ + --debug-e2e-test=$(DEBUG_E2E_TEST) \ + --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ + --uploader-type=$(UPLOADER_TYPE) \ + --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo @@ -166,31 +165,36 @@ run-e2e: ginkgo (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./e2e -- $(COMMON_ARGS) \ - -upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ - -upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \ - -migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \ - -migrate-from-velero-version=$(MIGRATE_FROM_VELERO_VERSION) \ - -additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \ - -additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ - -additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ - -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ - -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ - -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ - -default-cluster-context=$(DEFAULT_CLUSTER) \ - -standby-cluster-context=$(STANDBY_CLUSTER) \ - -snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ - -data-mover-plugin=$(DATA_MOVER_PLUGIN) \ - -standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \ - -standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \ - -standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \ - -default-cluster-name=$(DEFAULT_CLUSTER_NAME) \ - -standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ - -eks-policy-arn=$(EKS_POLICY_ARN) \ - -default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \ - -standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) - -kibishii-directory=$(KIBISHII_DIRECTORY) \ - -disable-informer-cache=$(DISABLE_INFORMER_CACHE) + @$(GINKGO) run \ + -v \ + --junit-report report.xml \ + --label-filter="$(GINKGO_LABELS)" \ + ./e2e \ + -- $(COMMON_ARGS) \ + --upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ + --upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \ + --migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \ + --migrate-from-velero-version=$(MIGRATE_FROM_VELERO_VERSION) \ + --additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \ + --additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ + --additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ + --additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ + --additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ + --additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ + --default-cluster-context=$(DEFAULT_CLUSTER) \ + --standby-cluster-context=$(STANDBY_CLUSTER) \ + --snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ + --data-mover-plugin=$(DATA_MOVER_PLUGIN) \ + --standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \ + --standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \ + --standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \ + --default-cluster-name=$(DEFAULT_CLUSTER_NAME) \ + --standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ + --eks-policy-arn=$(EKS_POLICY_ARN) \ + --default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \ + --standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) + --kibishii-directory=$(KIBISHII_DIRECTORY) \ + --disable-informer-cache=$(DISABLE_INFORMER_CACHE) .PHONY: run-perf run-perf: ginkgo @@ -200,20 +204,25 @@ run-perf: ginkgo (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./perf -- $(COMMON_ARGS) \ - -nfs-server-path=$(NFS_SERVER_PATH) \ - -test-case-describe=$(TEST_CASE_DESCRIBE) \ - -backup-for-restore=$(BACKUP_FOR_RESTORE) \ - -delete-cluster-resource=$(Delete_Cluster_Resource) \ - -node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \ - -node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \ - -node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \ - -node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \ - -velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \ - -velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \ - -velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \ - -velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \ - -pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT) + @$(GINKGO) run \ + -v \ + --junit-report report.xml \ + --label-filter="$(GINKGO_LABELS)" \ + ./perf \ + -- $(COMMON_ARGS) \ + --nfs-server-path=$(NFS_SERVER_PATH) \ + --test-case-describe=$(TEST_CASE_DESCRIBE) \ + --backup-for-restore=$(BACKUP_FOR_RESTORE) \ + --delete-cluster-resource=$(Delete_Cluster_Resource) \ + --node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \ + --node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \ + --node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \ + --node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \ + --velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \ + --velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \ + --velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \ + --velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \ + --pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/README.md b/test/e2e/README.md index b1d8259997..2e44cffe22 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -157,9 +157,9 @@ Basic examples: BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET= CREDS_FILE=/path/to/azure-creds CLOUD_PROVIDER=azure make test-e2e ``` Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container) -1. Run Ginko-focused Restore Multi-API Groups tests using Minio as the backup storage location: +1. Run Multi-API group and version tests using MinIO as the backup storage location: ```bash - BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=:9000" BSL_PREFIX= BSL_BUCKET= CREDS_FILE= CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws VELERO_NAMESPACE="velero" GINKGO_FOCUS="API group versions" make test-e2e + BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=:9000" BSL_PREFIX= BSL_BUCKET= CREDS_FILE= CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws VELERO_NAMESPACE="velero" GINKGO_LABELS="APIGroup && APIVersion" make test-e2e ``` 1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location: ```bash @@ -208,60 +208,66 @@ Migration examples: ``` -1. Datamover tests: +1. Data mover tests: - The example shows all essential `make` variables for a Datamover test which is migrate from a AKS cluster to a EKS cluster. + The example shows all essential `make` variables for a data mover test which is migrate from a AKS cluster to a EKS cluster. Note: STANDBY_CLUSTER_CLOUD_PROVIDER and STANDBY_CLUSTER_OBJECT_STORE_PROVIDER is essential here, it is for identify plugins to be installed on target cluster, since DEFAULT cluster's provider is different from STANDBY cluster, plugins are different as well. ```bash + CLOUD_PROVIDER=azure \ + DEFAULT_CLUSTER= \ + STANDBY_CLUSTER= \ + FEATURES=EnableCSI \ + OBJECT_STORE_PROVIDER=aws \ + CREDS_FILE= \ + BSL_CONFIG=region= \ + BSL_BUCKET= \ + BSL_PREFIX= \ + VSL_CONFIG=region= \ + SNAPSHOT_MOVE_DATA=true \ + STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ + STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ + GINKGO_LABELS="Migration" \ make test-e2e - CLOUD_PROVIDER=azure \ - DEFAULT_CLUSTER= \ - STANDBY_CLUSTER= \ - FEATURES=EnableCSI \ - OBJECT_STORE_PROVIDER=aws \ - CREDS_FILE= \ - BSL_CONFIG=region= \ - BSL_BUCKET= \ - BSL_PREFIX= \ - VSL_CONFIG=region= \ - SNAPSHOT_MOVE_DATA=true \ - STANDBY_CLUSTER_CLOUD_PROVIDER=aws \ - STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ - GINKGO_FOCUS=Migration ``` ## Filtering tests -Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo. +In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). +Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. -For filtering tests, using `make` variables `GINKGO_FOCUS` and `GINKGO_SKIP` : -1. `GINKGO_FOCUS`: Dot-separated list of labels to be included for Ginkgo description-based filtering. Optional. The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` `make` variable. This can be used to focus on specific tests. -1. `GINKGO_SKIP`: Dot-separated list of labels to be excluded for Ginkgo description-based filtering.Optional. The `-skip ` flag is passed to ginkgo using the `GINKGO_SKIP` `make` variable. This can be used to skip specific tests. +Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABELS` to filter test cases. +`GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels). -`GINKGO_FOCUS`/`GINKGO_SKIP` can be interpreted into multiple `-focus`/`-skip ` describe in [Description-Based Filtering](https://onsi.github.io/ginkgo/#description-based-filtering:~:text=Description%2DBased%20Filtering) by dot-separated format for test execution management please refer to examples below.: - - -For example, E2E tests can be run with specific cases to be included and/or excluded using the commands below: +### Examples +E2E tests can be run with specific cases to be included and/or excluded using the commands below: 1. Run Velero tests with specific cases to be included: ```bash + GINKGO_LABELS="Basic && Restic" \ + CLOUD_PROVIDER=aws \ + BSL_BUCKET=example-bucket \ + CREDS_FILE=/path/to/aws-creds \ make test-e2e \ - GINKGO_FOCUS =Basic\][\Restic \ - CLOUD_PROVIDER=aws BSL_BUCKET= BSL_PREFIX= CREDS_FILE=/path/to/aws-creds ``` - In this example, only case `[Basic][Restic]` is included. + In this example, only case have both `Basic` and `Restic` labels are included. 1. Run Velero tests with specific cases to be excluded: ```bash - make test-e2e \ - GINKGO_SKIP=Scale.Schedule.TTL.Upgrade\]\[Restic.Migration\][\Restic \ - CLOUD_PROVIDER=aws BSL_BUCKET= BSL_PREFIX= CREDS_FILE=/path/to/aws-creds + GINKGO_LABELS="Scale || Schedule || TTL || (Upgrade && Restic) || (Migration && Restic)" \ + CLOUD_PROVIDER=aws \ + BSL_BUCKET=example-bucket \ + CREDS_FILE=/path/to/aws-creds \ + make test-e2e ``` - In this example, case `Scale`, `Schedule`, `TTL`, `[Upgrade][Restic]` and `[Migration][Restic]` will be skipped. - - + In this example, cases are labelled as + * `Scale` + * `Schedule` + * `TTL` + * `Upgrade` and `Restic` + * `Migration` and `Restic` + will be skipped. ## Full Tests execution diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 4fc17ed975..7f67dbf6e1 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -53,7 +53,7 @@ type apiGropuVersionsTest struct { want map[string]map[string]string } -func APIGropuVersionsTest() { +func APIGroupVersionsTest() { var ( group string err error diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 45b093b13b..0d22510bc1 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -112,79 +112,126 @@ func init() { // caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test // cases can be executed as expected successful result. -var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest) -var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) +var _ = Describe("Velero tests with various CRD API group versions", + Label("APIGroup", "APIVersion", "SKIP_KIND"), APIGroupVersionsTest) +var _ = Describe("CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", + Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest) -// Test backup and restore of Kibishi using restic -var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) +// Test backup and restore of Kibishii using restic +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Basic", "Restic"), BackupRestoreWithRestic) -var _ = Describe("[Basic][Snapshot][SkipVanillaZfs] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "SkipVanillaZfs"), BackupRestoreWithSnapshots) -var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "RetainPV"), BackupRestoreRetainedPVWithSnapshots) -var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Restic", "RetainPV"), BackupRestoreRetainedPVWithRestic) -var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest) +var _ = Describe("Backup/restore of cluster resources", + Label("Basic", "ClusterResource"), ResourcesCheckTest) -var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore) +var _ = Describe("Service NodePort reservation during restore is configurable", + Label("Basic", "NodePort"), NodePortTest) -// Upgrade test by Kibishi using restic -var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) -var _ = Describe("[Upgrade][Snapshot][SkipVanillaZfs] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) +var _ = Describe("Storage class of persistent volumes and persistent volume claims can be changed during restores", + Label("Basic", "StorageClass"), StorageClasssChangingTest) -// test filter objects by namespace, type, or labels when backup or restore. -var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest) -var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Backup] Velero test on exclude namespace from the cluster backup", BackupWithExcludeNamespaces) -var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Restore] Velero test on exclude namespace from the cluster restore", RestoreWithExcludeNamespaces) -var _ = Describe("[ResourceFiltering][ExcludeResources][Backup] Velero test on exclude resources from the cluster backup", BackupWithExcludeResources) -var _ = Describe("[ResourceFiltering][ExcludeResources][Restore] Velero test on exclude resources from the cluster restore", RestoreWithExcludeResources) -var _ = Describe("[ResourceFiltering][IncludeNamespaces][Backup] Velero test on include namespace from the cluster backup", BackupWithIncludeNamespaces) -var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on include namespace from the cluster restore", RestoreWithIncludeNamespaces) -var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) -var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) -var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) -var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) - -// backup VolumeInfo test -var _ = Describe("[BackupVolumeInfo][SkippedVolume]", SkippedVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][FilesystemUpload]", FilesystemUploadVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][CSIDataMover]", CSIDataMoverVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][CSISnapshot]", CSISnapshotVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoTest) - -var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest) - -var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) -var _ = Describe("[Backups][Deletion][Snapshot][SkipVanillaZfs] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[Backups][TTL][LongTime][Snapshot][SkipVanillaZfs] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) -var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) - -var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) -var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) -var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest) +var _ = Describe("Node selectors of persistent volume claims can be changed during restores", + Label("Basic", "SelectedNode", "SKIP_KIND"), PVCSelectedNodeChangingTest) -var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) +var _ = Describe("Backup/restore of 2500 namespaces", + Label("Scale", "LongTime"), MultiNSBackupRestore) -var _ = Describe("[BSL][Deletion][Snapshot][SkipVanillaZfs] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) -var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) +// Upgrade test by Kibishii using Restic +var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Upgrade", "Restic"), BackupUpgradeRestoreWithRestic) +var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Upgrade", "Snapshot", "SkipVanillaZfs"), BackupUpgradeRestoreWithSnapshots) -var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic) -var _ = Describe("[Migration][Snapshot][SkipVanillaZfs] Migrate resources between clusters by snapshot", MigrationWithSnapshots) - -var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest) -var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) -var _ = Describe("[NamespaceMapping][Single][Snapshot][SkipVanillaZfs] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) -var _ = Describe("[NamespaceMapping][Multiple][Snapshot]SkipVanillaZfs] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) - -var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-In"), OptInPVBackupTest) -var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-Out"), OptOutPVBackupTest) - -var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest) -var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest) -var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest) +// test filter objects by namespace, type, or labels when backup or restore. +var _ = Describe("Resources with the label velero.io/exclude-from-backup=true are not included in backup", + Label("ResourceFiltering", "ExcludeFromBackup"), ExcludeFromBackupTest) +var _ = Describe("Velero test on exclude namespace from the cluster backup", + Label("ResourceFiltering", "ExcludeNamespaces", "Backup"), BackupWithExcludeNamespaces) +var _ = Describe("Velero test on exclude namespace from the cluster restore", + Label("ResourceFiltering", "ExcludeNamespaces", "Restore"), RestoreWithExcludeNamespaces) +var _ = Describe("Velero test on exclude resources from the cluster backup", + Label("ResourceFiltering", "ExcludeResources", "Backup"), BackupWithExcludeResources) +var _ = Describe("Velero test on exclude resources from the cluster restore", + Label("ResourceFiltering", "ExcludeResources", "Restore"), RestoreWithExcludeResources) +var _ = Describe("Velero test on include namespace from the cluster backup", + Label("ResourceFiltering", "IncludeNamespaces", "Backup"), BackupWithIncludeNamespaces) +var _ = Describe("Velero test on include namespace from the cluster restore", + Label("ResourceFiltering", "IncludeNamespaces", "Restore"), RestoreWithIncludeNamespaces) +var _ = Describe("Velero test on include resources from the cluster backup", + Label("ResourceFiltering", "IncludeResources", "Backup"), BackupWithIncludeResources) +var _ = Describe("Velero test on include resources from the cluster restore", + Label("ResourceFiltering", "IncludeResources", "Restore"), RestoreWithIncludeResources) +var _ = Describe("Velero test on backup include resources matching the label selector", + Label("ResourceFiltering", "LabelSelector"), BackupWithLabelSelector) +var _ = Describe("Velero test on skip backup of volume by resource policies", + Label("ResourceFiltering", "ResourcePolicies", "Restic"), ResourcePoliciesTest) -var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest) -var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest) +// backup VolumeInfo test +var _ = Describe("", Label("BackupVolumeInfo", "SkippedVolume"), SkippedVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "FilesystemUpload"), FilesystemUploadVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "CSIDataMover"), CSIDataMoverVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "CSISnapshot"), CSISnapshotVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "NativeSnapshot"), NativeSnapshotVolumeInfoTest) + +var _ = Describe("Velero test on resource modifiers from the cluster restore", + Label("ResourceModifier", "Restore"), ResourceModifiersTest) + +var _ = Describe("Velero tests of Restic backup deletion", + Label("Backups", "Deletion", "Restic"), BackupDeletionWithRestic) +var _ = Describe("Velero tests of snapshot backup deletion", + Label("Backups", "Deletion", "Snapshot", "SkipVanillaZfs"), BackupDeletionWithSnapshots) +var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), TTLTest) +var _ = Describe("Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", + Label("Backups", "BackupsSync"), BackupsSyncTest) + +var _ = Describe("Backup will be created periodically by schedule defined by a Cron expression", + Label("Schedule", "BR", "Pause", "LongTime"), ScheduleBackupTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("Schedule", "OrderedResources"), ScheduleOrderedResources) +var _ = Describe("Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", + Label("Schedule", "BackupCreation", "SKIP_KIND"), ScheduleBackupCreationTest) + +var _ = Describe("Velero test on ssr object when controller namespace mix-ups", + Label("PrivilegesMgmt", "SSR"), SSRTest) + +var _ = Describe("Local backups will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Snapshot", "SkipVanillaZfs"), BslDeletionWithSnapshots) +var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Restic"), BslDeletionWithRestic) + +var _ = Describe("Migrate resources between clusters by Restic", + Label("Migration", "Restic"), MigrationWithRestic) +var _ = Describe("Migrate resources between clusters by snapshot", + Label("Migration", "Snapshot", "SkipVanillaZfs"), MigrationWithSnapshots) + +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Restic"), OneNamespaceMappingResticTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Restic"), MultiNamespacesMappingResticTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Snapshot", "SkipVanillaZfs"), OneNamespaceMappingSnapshotTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Snapshot", "SkipVanillaZfs"), MultiNamespacesMappingSnapshotTest) + +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptIn"), OptInPVBackupTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptOut"), OptOutPVBackupTest) + +var _ = Describe("Velero test on parallel files upload", + Label("UploaderConfig", "ParallelFilesUpload"), ParallelFilesUploadTest) +var _ = Describe("Velero test on parallel files download", + Label("UploaderConfig", "ParallelFilesDownload"), ParallelFilesDownloadTest) func GetKubeconfigContext() error { var err error diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index a2aa85d63c..e047749c22 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -94,11 +94,14 @@ func initConfig() error { return nil } -var _ = Describe("[PerformanceTest][BackupAndRestore] Velero test on both backup and restore resources", test.TestFunc(&basic.BasicTest{})) +var _ = Describe("Velero test on both backup and restore resources", + Label("PerformanceTest", "BackupAndRestore"), test.TestFunc(&basic.BasicTest{})) -var _ = Describe("[PerformanceTest][Backup] Velero test on only backup resources", test.TestFunc(&backup.BackupTest{})) +var _ = Describe("Velero test on only backup resources", + Label("PerformanceTest", "Backup"), test.TestFunc(&backup.BackupTest{})) -var _ = Describe("[PerformanceTest][Restore] Velero test on only restore resources", test.TestFunc(&restore.RestoreTest{})) +var _ = Describe("Velero test on only restore resources", + Label("PerformanceTest", "Restore"), test.TestFunc(&restore.RestoreTest{})) func TestE2e(t *testing.T) { flag.Parse()