From 4e67c1eec7adc2c0ddc5c7f8dd57771d8db82ff9 Mon Sep 17 00:00:00 2001 From: James Lu Date: Sat, 2 Dec 2023 04:22:50 +0800 Subject: [PATCH] feat(backup): AWS IAM Role Annotation Move AWS IAM Role Annotation implementations to datastore/kubernetes.go and trigger it by backup target validator. Ref: 5411, 6947 Signed-off-by: James Lu --- datastore/kubernetes.go | 89 +++++++++++++++++++++ webhook/resources/backuptarget/validator.go | 44 ++++++++-- 2 files changed, 127 insertions(+), 6 deletions(-) diff --git a/datastore/kubernetes.go b/datastore/kubernetes.go index 5b00d351a0..d3d807150c 100644 --- a/datastore/kubernetes.go +++ b/datastore/kubernetes.go @@ -692,6 +692,95 @@ func (s *DataStore) UpdateSecret(namespace string, secret *corev1.Secret) (*core return s.kubeClient.CoreV1().Secrets(namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) } +// HandleSecretsForAWSIAMRoleAnnotation handles AWS IAM Role Annotation when a BackupTarget is created or updated with a Secret. +func (s *DataStore) HandleSecretsForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, newSecretName string, isBackupTargetURLChanged bool) (err error) { + isSameSecretName := oldSecretName == newSecretName + if isSameSecretName && !isBackupTargetURLChanged { + return nil + } + + isArnExists := false + if !isSameSecretName { + isArnExists, _, err = s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, oldSecretName, true) + if err != nil { + return err + } + } + _, isValidSecret, err := s.updateSecretForAWSIAMRoleAnnotation(backupTargetURL, newSecretName, false) + if err != nil { + return err + } + // kubernetes_secret_controller will not reconcile the secret that does not exist such as named "", so we remove AWS IAM role annotation of pods if new secret name is "". + if !isValidSecret && isArnExists { + if err = s.removePodsAWSIAMRoleAnnotation(); err != nil { + return err + } + } + return nil +} + +// updateSecretForAWSIAMRoleAnnotation adds the AWS IAM Role annotation to make an update to reconcile in kubernetes secret controller and returns +// +// isArnExists = true if annotation had been added to the secret for first parameter, +// isValidSecret = true if this secret is valid for second parameter. +// err != nil if there is an error occurred. +func (s *DataStore) updateSecretForAWSIAMRoleAnnotation(backupTargetURL, secretName string, isOldSecret bool) (isArnExists bool, isValidSecret bool, err error) { + if secretName == "" { + return false, false, nil + } + + secret, err := s.GetSecret(s.namespace, secretName) + if err != nil { + if apierrors.IsNotFound(err) { + return false, false, nil + } + return false, false, err + } + + if isOldSecret { + delete(secret.Annotations, types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget))) + isArnExists = secret.Data[types.AWSIAMRoleArn] != nil + } else { + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[types.GetLonghornLabelKey(string(types.LonghornLabelBackupTarget))] = backupTargetURL + } + + if _, err = s.UpdateSecret(s.namespace, secret); err != nil { + return false, false, err + } + return isArnExists, true, nil +} + +func (s *DataStore) removePodsAWSIAMRoleAnnotation() error { + managerPods, err := s.ListManagerPods() + if err != nil { + return err + } + + instanceManagerPods, err := s.ListInstanceManagerPods() + if err != nil { + return err + } + pods := append(managerPods, instanceManagerPods...) + + for _, pod := range pods { + _, exist := pod.Annotations[types.AWSIAMRoleAnnotation] + if !exist { + continue + } + delete(pod.Annotations, types.AWSIAMRoleAnnotation) + if _, err := s.UpdatePod(pod); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + } + return nil +} + // DeleteSecret deletes a Secret resource func (s *DataStore) DeleteSecret(namespace, name string) error { return s.kubeClient.CoreV1().Secrets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) diff --git a/webhook/resources/backuptarget/validator.go b/webhook/resources/backuptarget/validator.go index 9b70da4149..ad90d92e1c 100644 --- a/webhook/resources/backuptarget/validator.go +++ b/webhook/resources/backuptarget/validator.go @@ -68,6 +68,10 @@ func (b *backupTargetValidator) Create(request *admission.Request, newObj runtim return werror.NewInvalidError(err.Error(), "") } + if err := b.handleAWSIAMRoleAnnotation(backupTarget, nil); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + return nil } @@ -78,12 +82,6 @@ func (b *backupTargetValidator) Update(request *admission.Request, oldObj runtim urlChanged := oldBackupTarget.Spec.BackupTargetURL != newBackupTarget.Spec.BackupTargetURL secretChanged := oldBackupTarget.Spec.CredentialSecret != newBackupTarget.Spec.CredentialSecret - if urlChanged || secretChanged { - if err := b.validateDRVolume(newBackupTarget); err != nil { - return werror.NewInvalidError(err.Error(), "") - } - } - if urlChanged { if err := b.validateBackupTargetURL(newBackupTarget.Spec.BackupTargetURL); err != nil { return werror.NewInvalidError(err.Error(), "") @@ -96,6 +94,16 @@ func (b *backupTargetValidator) Update(request *admission.Request, oldObj runtim } } + if urlChanged || secretChanged { + if err := b.validateDRVolume(newBackupTarget); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + + if err := b.handleAWSIAMRoleAnnotation(newBackupTarget, oldBackupTarget); err != nil { + return werror.NewInvalidError(err.Error(), "") + } + } + return nil } @@ -219,6 +227,30 @@ func (b *backupTargetValidator) validateCredentialSecret(secretName string) erro return nil } +func (b *backupTargetValidator) handleAWSIAMRoleAnnotation(newBackupTarget, oldBackupTarget *longhorn.BackupTarget) error { + oldBackupTargetURL := "" + oldBackupTargetSecret := "" + if oldBackupTarget != nil { + oldBackupTargetURL = oldBackupTarget.Spec.BackupTargetURL + oldBackupTargetSecret = oldBackupTarget.Spec.CredentialSecret + } + uOld, err := url.Parse(oldBackupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", oldBackupTargetURL) + } + newBackupTargetURL := newBackupTarget.Spec.BackupTargetURL + u, err := url.Parse(newBackupTargetURL) + if err != nil { + return errors.Wrapf(err, "failed to parse %v as url", newBackupTargetURL) + } + if u.Scheme == types.BackupStoreTypeS3 || (uOld.Scheme == types.BackupStoreTypeS3 && newBackupTargetURL == "") { + if err := b.ds.HandleSecretsForAWSIAMRoleAnnotation(newBackupTargetURL, oldBackupTargetSecret, newBackupTarget.Spec.CredentialSecret, oldBackupTargetURL != newBackupTargetURL); err != nil { + return err + } + } + return nil +} + func (b *backupTargetValidator) validateDRVolume(backupTarget *longhorn.BackupTarget) error { vs, err := b.ds.ListDRVolumesRO() if err != nil {