K8SPSMDB-1213 Add arm64 support for the e2e tests

e2e-tests/arbiter/run

@@ -74,9 +74,8 @@ main() {
 	deploy_cert_manager
 
 	desc 'create secrets and start client'
-	kubectl_bin apply \
-		-f $conf_dir/client.yml \
-		-f $conf_dir/secrets.yml
+	kubectl_bin apply -f $conf_dir/secrets.yml
+	apply_client $conf_dir/client.yml
 
 	desc 'check arbiter without service-per-pod'
 	check_cr_config "arbiter-rs0"

e2e-tests/balancer/run

@@ -69,9 +69,9 @@ main() {
 
 	desc 'create first PSMDB cluster'
 	cluster="some-name"
-	kubectl_bin apply \
-		-f "$conf_dir/secrets.yml" \
-		-f "$conf_dir/client-70.yml"
+  
+	kubectl_bin apply -f $conf_dir/secrets.yml
+	apply_client $conf_dir/client-70.yml
 
 	if version_gt "1.19" && [ $EKS -ne 1 ]; then
 		$sed 's/docker/runc/g' "$conf_dir/container-rc.yaml" | kubectl_bin apply -f -

e2e-tests/conf/client.yml

@@ -15,7 +15,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
         - name: psmdb-client
-          image: percona/percona-server-mongodb:4.4
+          image: percona/percona-server-mongodb:4.4-multi
           imagePullPolicy: Always
           command:
           - sleep

e2e-tests/conf/client_with_tls.yml

@@ -15,7 +15,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
         - name: psmdb-client
-          image: percona/percona-server-mongodb:4.4
+          image: percona/percona-server-mongodb:4.4-multi
           imagePullPolicy: Always
           command: ["/bin/bash","-c","cat /etc/mongodb-ssl/tls.key /etc/mongodb-ssl/tls.crt > /tmp/tls.pem && sleep 100500"]
           volumeMounts:

e2e-tests/cross-site-sharded/run

@@ -39,9 +39,8 @@ desc "create main cluster"
 create_infra "$namespace"
 
 desc 'create secrets and start client'
-kubectl_bin apply \
-	-f "$conf_dir/client.yml" \
-	-f "$test_dir/conf/secrets.yml"
+kubectl_bin apply -f $test_dir/conf/secrets.yml
+apply_client $conf_dir/client.yml
 
 desc "create main PSMDB cluster $main_cluster."
 apply_cluster "$test_dir/conf/$main_cluster.yml"
@@ -112,8 +111,7 @@ create_namespace $replica_namespace 0
 deploy_operator
 
 desc 'start client'
-kubectl_bin apply \
-	-f "$conf_dir/client.yml"
+apply_client $conf_dir/client.yml
 
 desc "copy secrets from main to replica namespace and create all of them"
 kubectl get secret ${main_cluster}-secrets -o yaml -n ${namespace} \

e2e-tests/custom-replset-name/conf/some-name.yml

@@ -6,7 +6,7 @@ spec:
   crVersion: 1.18.0
   backup:
     enabled: true
-    image: percona/percona-backup-mongodb:2.0.4
+    image:
     pitr:
       enabled: false
     serviceAccountName: percona-server-mongodb-operator
@@ -33,7 +33,7 @@ spec:
           bucket: operator-testing
           prefix: psmdb
           endpointUrl: https://storage.googleapis.com
-  image: percona/percona-server-mongodb:4.4.10-11
+  image:
   imagePullPolicy: Always
   pmm:
     enabled: false

e2e-tests/custom-replset-name/run

@@ -10,7 +10,11 @@ create_infra $namespace
 apply_s3_storage_secrets
 deploy_minio
 
-kubectl_bin apply -f $conf_dir/secrets.yml -f $conf_dir/client.yml -f $conf_dir/minio-secret.yml
+desc 'create secrets and start client'
+kubectl_bin apply -f $conf_dir/secrets.yml
+kubectl_bin apply -f $conf_dir/minio-secret.yml
+apply_client $conf_dir/client.yml
+
 cluster="some-name"
 
 desc 'create first PSMDB cluster'

e2e-tests/custom-tls/run

@@ -32,8 +32,8 @@ main() {
 	destroy_cert_manager || true # We need to be sure that we are getting certificates created by the operator, not by cert-manager
 
 	desc 'create secrets and start client'
-	kubectl_bin apply -f "$conf_dir/secrets.yml"
-	kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
+	kubectl_bin apply -f $conf_dir/secrets.yml
+	apply_client $conf_dir/client_with_tls.yml
 
 	cluster="some-name"
 	desc "create first PSMDB cluster $cluster"

e2e-tests/custom-users-roles-sharded/run

@@ -79,10 +79,9 @@ create_infra "$namespace"
 mongosUri="userAdmin:userAdmin123456@$cluster-mongos.$namespace"
 
 desc 'create secrets and start client'
-kubectl_bin apply -f "${conf_dir}/client.yml" \
-	-f "${conf_dir}/secrets.yml" \
-	-f "${test_dir}/conf/app-user-secrets.yml"
-
+kubectl_bin apply -f $conf_dir/secrets.yml
+kubectl_bin apply -f $test_dir/conf/app-user-secrets.yml
+apply_client $conf_dir/client.yml
 
 apply_s3_storage_secrets
 if version_gt "1.19" && [ $EKS -ne 1 ]; then
@@ -135,7 +134,7 @@ kubectl_bin patch psmdb ${cluster} --type=merge --patch '{
 					"key": "userTwoPassKey"
 				},
 				"roles": [
-					{"db":"admin","name":"userAdminAnyDatabase"}, 
+					{"db":"admin","name":"userAdminAnyDatabase"},
 					{"db":"admin","name":"clusterAdmin"}
 				]
 			}

e2e-tests/custom-users-roles/run

@@ -68,9 +68,9 @@ cluster="some-name-rs0"
 create_infra $namespace
 
 desc 'create secrets and start client'
-kubectl_bin apply -f "${conf_dir}/client.yml" \
-	-f "${conf_dir}/secrets.yml" \
-	-f "${test_dir}/conf/app-user-secrets.yml"
+kubectl_bin apply -f $conf_dir/secrets.yml
+kubectl_bin apply -f $test_dir/conf/app-user-secrets.yml
+apply_client $conf_dir/client.yml
 
 mongoUri="userAdmin:userAdmin123456@$cluster.$namespace"
 
@@ -107,7 +107,7 @@ kubectl_bin patch psmdb ${psmdb} --type=merge --patch '{
 					"key": "userTwoPassKey"
 				},
 				"roles": [
-					{"db":"admin","name":"userAdminAnyDatabase"}, 
+					{"db":"admin","name":"userAdminAnyDatabase"},
 					{"db":"admin","name":"clusterAdmin"}
 				]
 			}

e2e-tests/data-at-rest-encryption/run

@@ -13,7 +13,8 @@ deploy_minio
 apply_s3_storage_secrets
 
 desc 'create secrets and start client'
-kubectl_bin apply -f "$conf_dir/secrets.yml" -f "$conf_dir/client.yml"
+kubectl_bin apply -f $conf_dir/secrets.yml
+apply_client $conf_dir/client.yml
 
 cluster='some-name'
 desc "create PSMDB cluster $cluster"
@@ -57,10 +58,7 @@ sleep 5
 
 desc "check backup and restore -- minio"
 backup_dest_minio=$(get_backup_dest "$backup_name_minio")
-retry 3 8 kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
-	/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
-	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://${backup_dest_minio}/rs0/ \
-	| grep myApp.test.gz
+retry 3 8 aws_cli "s3 ls s3://${backup_dest_minio}/rs0/" | grep "myApp.test.gz"
 run_mongos 'use myApp\n db.test.insert({ x: 100501 })' "myApp:myPass@$cluster-mongos.$namespace"
 compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "-2nd"
 run_restore "$backup_name_minio"

e2e-tests/data-sharded/run

@@ -39,8 +39,8 @@ main() {
 	deploy_cert_manager
 
 	desc 'create secrets and start client'
-	kubectl_bin apply -f "$conf_dir/secrets.yml"
-	kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
+	kubectl_bin apply -f $conf_dir/secrets.yml
+	apply_client $conf_dir/client_with_tls.yml
 
 	cluster="some-name"
 	desc "create first PSMDB cluster $cluster"

e2e-tests/default-cr/run

@@ -48,26 +48,46 @@ function main() {
 
 	desc 'create secrets and start client'
 	kubectl_bin apply -f $deploy_dir/secrets.yaml
-	kubectl_bin apply -f $conf_dir/client.yml
+	apply_client $conf_dir/client.yml
 
 	desc "create first PSMDB cluster $cluster"
 	kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} --server-side --force-conflicts -f $deploy_dir/crd.yaml
+
+
+ 	local temp_operator_yaml="$(mktemp)"
+
 	if [ -n "$OPERATOR_NS" ]; then
 		apply_rbac cw-rbac
 		kubectl_bin apply -n ${OPERATOR_NS} -f $deploy_dir/cw-operator.yaml
 	else
 		apply_rbac rbac
-		yq eval '((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true")' "$deploy_dir/operator.yaml" \
-			| kubectl_bin apply -f -
+		yq eval '((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true")' "$deploy_dir/operator.yaml" > $temp_operator_yaml
+
+		if [[ $ARCH == "arm64" ]]; then
+			yq eval -i '(.spec.template.spec.tolerations += '"$TOLERATIONS_ARM64"')' $temp_operator_yaml
+		fi
+
+		kubectl_bin apply -f  $temp_operator_yaml
 	fi
 
+	local temp_cr="$(mktemp)"
 	yq eval '.spec.upgradeOptions.versionServiceEndpoint = "https://check-dev.percona.com" |
 		.spec.replsets[].affinity.antiAffinityTopologyKey = "none" |
 		.spec.replsets[].nonvoting.affinity.antiAffinityTopologyKey = "none" |
 		.spec.replsets[].arbiter.affinity.antiAffinityTopologyKey = "none" |
 		.spec.sharding.configsvrReplSet.affinity.antiAffinityTopologyKey = "none" |
-		.spec.sharding.mongos.affinity.antiAffinityTopologyKey = "none"' $deploy_dir/cr.yaml \
-		| kubectl_bin apply -f -
+		.spec.sharding.mongos.affinity.antiAffinityTopologyKey = "none"' $deploy_dir/cr.yaml > $temp_cr
+
+	if [[ $ARCH == "arm64" ]]; then
+        yq eval '.spec.replsets[].tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec | select(has("sharding"))).sharding.configsvrReplSet.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec | select(has("sharding"))).sharding.mongos.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec.replsets[] | select(has("arbiter"))).arbiter.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec.replsets[] | select(has("nonvoting"))).nonvoting.tolerations += '"$TOLERATIONS_ARM64"'' $temp_cr |
+			kubectl_bin apply -f -
+	else
+		kubectl_bin apply -f $temp_cr
+	fi
 
 	desc 'check if all 3 Pods started'
 	wait_cluster_consistency $cluster 70
@@ -137,7 +157,20 @@ function main() {
 	cluster="minimal-cluster"
 	yq eval '.metadata.name = "'${cluster}'"' $deploy_dir/secrets.yaml | kubectl_bin apply -f -
 
-	yq eval '.spec.upgradeOptions.versionServiceEndpoint = "https://check-dev.percona.com"' $deploy_dir/cr-minimal.yaml | kubectl_bin apply -f -
+	local temp_cr_minimal="$(mktemp)"
+	yq eval '.spec.upgradeOptions.versionServiceEndpoint = "https://check-dev.percona.com"' $deploy_dir/cr-minimal.yaml > $temp_cr_minimal
+
+	if [[ $ARCH == "arm64" ]]; then
+        yq eval '.spec.replsets[].tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec | select(has("sharding"))).sharding.configsvrReplSet.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec | select(has("sharding"))).sharding.mongos.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec.replsets[] | select(has("arbiter"))).arbiter.tolerations += '"$TOLERATIONS_ARM64"' |
+			(.spec.replsets[] | select(has("nonvoting"))).nonvoting.tolerations += '"$TOLERATIONS_ARM64"'' $temp_cr_minimal |
+			kubectl_bin apply -f -
+	else
+		kubectl_bin apply -f $temp_cr_minimal
+	fi
+
 	desc 'check if all Pods started'
 	wait_cluster_consistency "${cluster}"
 

e2e-tests/demand-backup-eks-credentials/run

@@ -14,9 +14,8 @@ fi
 create_infra $namespace
 
 desc 'create secrets and start client'
-kubectl_bin apply \
-	-f "$conf_dir/secrets.yml" \
-	-f "$conf_dir/client.yml"
+kubectl_bin apply -f $conf_dir/secrets.yml
+apply_client $conf_dir/client.yml
 
 cluster="some-name-rs0"
 desc "create first PSMDB cluster $cluster"

e2e-tests/demand-backup-physical-sharded/run

@@ -66,11 +66,13 @@ apply_s3_storage_secrets
 ### Case 1: Backup and restore on sharded cluster
 desc 'Testing on sharded cluster'
 
-echo "Creating PSMDB cluster"
+desc 'create secrets and start client'
+kubectl_bin apply -f $conf_dir/secrets.yml
+apply_client $conf_dir/client_with_tls.yml
+
 cluster="some-name"
-kubectl_bin apply -f "${conf_dir}/secrets.yml"
-apply_cluster "${test_dir}/conf/${cluster}-sharded.yml"
-kubectl_bin apply -f "${conf_dir}/client_with_tls.yml"
+desc "create first PSMDB cluster $cluster"
+apply_cluster $test_dir/conf/$cluster-sharded.yml
 
 echo "check if all pods started"
 wait_for_running ${cluster}-rs0 3

e2e-tests/demand-backup-physical/run

@@ -58,11 +58,13 @@ apply_s3_storage_secrets
 
 desc 'Testing on not sharded cluster'
 
+desc 'create secrets and start client'
+kubectl_bin apply -f $test_dir/conf/secrets.yml
+apply_client $conf_dir/client_with_tls.yml
+
 echo "Creating PSMDB cluster"
 cluster="some-name"
-kubectl_bin apply -f "${test_dir}/conf/secrets.yml"
-apply_cluster "${test_dir}/conf/${cluster}.yml"
-kubectl_bin apply -f "${conf_dir}/client_with_tls.yml"
+apply_cluster $test_dir/conf/$cluster.yml
 
 echo "check if all pods started"
 wait_for_running ${cluster}-rs0 3

e2e-tests/demand-backup-sharded/run

@@ -19,11 +19,9 @@ create_infra "$namespace"
 
 deploy_minio
 
-desc 'create first PSMDB cluster'
-cluster="some-name"
-kubectl_bin apply \
-	-f "$conf_dir/secrets.yml" \
-	-f "$conf_dir/client.yml"
+desc 'create secrets and start client'
+kubectl_bin apply -f $conf_dir/secrets.yml
+apply_client $conf_dir/client.yml
 
 apply_s3_storage_secrets
 if version_gt "1.19" && [ $EKS -ne 1 ]; then
@@ -34,6 +32,8 @@ else
 	kubectl_bin apply -f "$conf_dir/container-rc.yaml"
 fi
 
+desc 'create first PSMDB cluster'
+cluster="some-name"
 apply_cluster "$test_dir/conf/$cluster-rs0.yml"
 desc 'check if all 3 Pods started'
 wait_for_running $cluster-rs0 3
@@ -146,10 +146,18 @@ fi
 
 desc 'check backup and restore -- minio'
 backup_dest_minio=$(get_backup_dest "$backup_name_minio")
-kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
-	/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
-	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls "s3://${backup_dest_minio}/rs0/" \
-	| grep "myApp.test.gz"
+
+retry=0
+until aws_cli "s3 ls s3://$backup_dest_minio/rs0/" | grep "myApp.test.gz"; do
+	if [[ $retry -ge 10 ]]; then
+		echo "Max retry count $retry reached. File myApp.test.gz wasn't found on s3://$backup_dest_minio/rs0/"
+		exit 1
+	fi
+	((retry += 1))
+	echo -n .
+	sleep 5
+done
+
 insert_data_mongos "100501" "myApp"
 insert_data_mongos "100501" "myApp1"
 insert_data_mongos "100501" "myApp2"
@@ -161,10 +169,7 @@ check_data
 desc 'delete backup and check if it is removed from bucket -- minio'
 kubectl_bin delete psmdb-backup --all
 
-backup_exists=$(kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
-	/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
-	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ \
-	| grep -c ${backup_dest_minio}_ | cat)
+backup_exists=$(aws_cli "s3 ls s3://operator-testing/" | grep -c ${backup_dest_minio}_ | cat)
 if [[ $backup_exists -eq 1 ]]; then
 	echo "Backup was not removed from bucket -- minio"
 	exit 1