feat(backup): AWS IAM Role Annotation

Move AWS IAM Role Annotation implementations to
datastore/kubernetes.go and trigger it by backup target validator.

Ref: 5411, 6947

Signed-off-by: James Lu <james.lu@suse.com>

datastore/kubernetes.go

@@ -692,6 +692,95 @@ func (s *DataStore) UpdateSecret(namespace string, secret *corev1.Secret) (*core
 	return s.kubeClient.CoreV1().Secrets(namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
 }
 
+// HandleSecretsForAWSIAMRoleAnnotation handles AWS IAM Role Annotation when a BackupTarget is created or updated with a Secret.
+func (s *DataStore) HandleSecretsForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, newSecretName string, isBackupTargetURLChanged bool) (err error) {
+	isSameSecretName := oldSecretName == newSecretName
+	if isSameSecretName && !isBackupTargetURLChanged {
+		return nil
+	}
+
+	isArnExists := false
+	if !isSameSecretName {
+		isArnExists, _, err = s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, true)
+		if err != nil {
+			return err
+		}
+	}
+	_, isValidSecret, err := s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, newSecretName, false)
+	if err != nil {
+		return err
+	}
+	// kubernetes_secret_controller will not reconcile the secret that does not exist such as named "", so we remove AWS IAM role annotation of pods if new secret name is "".
+	if !isValidSecret && isArnExists {
+		if err = s.removePodsAWSIAMRoleAnnotation(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// updateSecretForAWSIAMRoleAnnotation adds the AWS IAM Role annotation to make an update to reconcile in kubernetes secret controller and returns
+//
+//	isArnExists = true if annotation had been added to the secret for first parameter,
+//	isValidSecret = true if this secret is valid for second parameter.
+//	err != nil if there is an error occurred.
+func (s *DataStore) updateSecretForAWSIAMRoleAnnotation(backupTargetURL, secretName string, isOldSecret bool) (isArnExists bool, isValidSecret bool, err error) {
+	if secretName == "" {
+		return false, false, nil
+	}
+
+	secret, err := s.GetSecret(s.namespace, secretName)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, false, nil
+		}
+		return false, false, err
+	}
+
+	if isOldSecret {
+		delete(secret.Annotations, types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget)))
+		isArnExists = secret.Data[types.AWSIAMRoleArn] != nil
+	} else {
+		if secret.Annotations == nil {
+			secret.Annotations = make(map[string]string)
+		}
+		secret.Annotations[types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget))] = backupTargetURL
+	}
+
+	if _, err = s.UpdateSecret(s.namespace, secret); err != nil {
+		return false, false, err
+	}
+	return isArnExists, true, nil
+}
+
+func (s *DataStore) removePodsAWSIAMRoleAnnotation() error {
+	managerPods, err := s.ListManagerPods()
+	if err != nil {
+		return err
+	}
+
+	instanceManagerPods, err := s.ListInstanceManagerPods()
+	if err != nil {
+		return err
+	}
+	pods := append(managerPods, instanceManagerPods...)
+
+	for _, pod := range pods {
+		_, exist := pod.Annotations[types.AWSIAMRoleAnnotation]
+		if !exist {
+			continue
+		}
+		delete(pod.Annotations, types.AWSIAMRoleAnnotation)
+		if _, err := s.UpdatePod(pod); err != nil {
+			if apierrors.IsNotFound(err) {
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}
+
 // DeleteSecret deletes a Secret resource
 func (s *DataStore) DeleteSecret(namespace, name string) error {
 	return s.kubeClient.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})

webhook/resources/backuptarget/validator.go

@@ -68,6 +68,10 @@ func (b *backupTargetValidator) Create(request *admission.Request, newObj runtim
 		return werror.NewInvalidError(err.Error(), "")
 	}
 
+	if err := b.handleAWSIAMRoleAnnotation(backupTarget, nil); err != nil {
+		return werror.NewInvalidError(err.Error(), "")
+	}
+
 	return nil
 }
 
@@ -78,12 +82,6 @@ func (b *backupTargetValidator) Update(request *admission.Request, oldObj runtim
 	urlChanged := oldBackupTarget.Spec.BackupTargetURL != newBackupTarget.Spec.BackupTargetURL
 	secretChanged := oldBackupTarget.Spec.CredentialSecret != newBackupTarget.Spec.CredentialSecret
 
-	if urlChanged || secretChanged {
-		if err := b.validateDRVolume(newBackupTarget); err != nil {
-			return werror.NewInvalidError(err.Error(), "")
-		}
-	}
-
 	if urlChanged {
 		if err := b.validateBackupTargetURL(newBackupTarget.Spec.BackupTargetURL); err != nil {
 			return werror.NewInvalidError(err.Error(), "")
@@ -96,6 +94,16 @@ func (b *backupTargetValidator) Update(request *admission.Request, oldObj runtim
 		}
 	}
 
+	if urlChanged || secretChanged {
+		if err := b.validateDRVolume(newBackupTarget); err != nil {
+			return werror.NewInvalidError(err.Error(), "")
+		}
+
+		if err := b.handleAWSIAMRoleAnnotation(newBackupTarget, oldBackupTarget); err != nil {
+			return werror.NewInvalidError(err.Error(), "")
+		}
+	}
+
 	return nil
 }
 
@@ -219,6 +227,30 @@ func (b *backupTargetValidator) validateCredentialSecret(secretName string) erro
 	return nil
 }
 
+func (b *backupTargetValidator) handleAWSIAMRoleAnnotation(newBackupTarget, oldBackupTarget *longhorn.BackupTarget) error {
+	oldBackupTargetURL := ""
+	oldBackupTargetSecret := ""
+	if oldBackupTarget != nil {
+		oldBackupTargetURL = oldBackupTarget.Spec.BackupTargetURL
+		oldBackupTargetSecret = oldBackupTarget.Spec.CredentialSecret
+	}
+	uOld, err := url.Parse(oldBackupTargetURL)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse %v as url", oldBackupTargetURL)
+	}
+	newBackupTargetURL := newBackupTarget.Spec.BackupTargetURL
+	u, err := url.Parse(newBackupTargetURL)
+	if err != nil {
+		return errors.Wrapf(err, "failed to parse %v as url", newBackupTargetURL)
+	}
+	if u.Scheme == types.BackupStoreTypeS3 || (uOld.Scheme == types.BackupStoreTypeS3 && newBackupTargetURL == "") {
+		if err := b.ds.HandleSecretsForAWSIAMRoleAnnotation(newBackupTargetURL, oldBackupTargetSecret, newBackupTarget.Spec.CredentialSecret, oldBackupTargetURL != newBackupTargetURL); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (b *backupTargetValidator) validateDRVolume(backupTarget *longhorn.BackupTarget) error {
 	vs, err := b.ds.ListDRVolumesRO()
 	if err != nil {