From d314308d3af42e1d6185758ed6a91c268d590a75 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 27 Jul 2023 17:14:56 -0400 Subject: [PATCH 1/5] feat: Add immediate option to `ReplicationSource` Signed-off-by: Devin Buhl --- api/v1alpha1/replicationsource_types.go | 3 +++ api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ bundle/manifests/volsync.backube_replicationsources.yaml | 4 ++++ bundle/manifests/volsync.clusterserviceversion.yaml | 2 +- config/crd/bases/volsync.backube_replicationsources.yaml | 4 ++++ controllers/mover/rclone/builder.go | 3 +++ controllers/mover/rclone/mover.go | 4 ++++ controllers/mover/restic/builder.go | 3 +++ controllers/mover/restic/mover.go | 4 ++++ controllers/mover/rsync/builder.go | 3 +++ controllers/mover/rsync/mover.go | 4 ++++ controllers/mover/rsynctls/builder.go | 3 +++ controllers/mover/rsynctls/mover.go | 4 ++++ controllers/mover/syncthing/builder.go | 1 + controllers/mover/syncthing/mover.go | 1 + .../templates/volsync.backube_replicationsources.yaml | 3 +++ 16 files changed, 50 insertions(+), 1 deletion(-) diff --git a/api/v1alpha1/replicationsource_types.go b/api/v1alpha1/replicationsource_types.go index 8112e0580..ae0757f22 100644 --- a/api/v1alpha1/replicationsource_types.go +++ b/api/v1alpha1/replicationsource_types.go @@ -283,6 +283,9 @@ type ReplicationSourceSpec struct { // paused can be used to temporarily stop replication. Defaults to "false". //+optional Paused bool `json:"paused,omitempty"` + // immediate can be used to stop replication when a new ReplicationSource is created. Defaults to "true". + //+optional + Immediate *bool `json:"immediate,omitempty"` } type ReplicationSourceRsyncStatus struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fd9dd489e..a436d696e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1001,6 +1001,11 @@ func (in *ReplicationSourceSpec) DeepCopyInto(out *ReplicationSourceSpec) { *out = new(ReplicationSourceExternalSpec) (*in).DeepCopyInto(*out) } + if in.Immediate != nil { + in, out := &in.Immediate, &out.Immediate + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicationSourceSpec. diff --git a/bundle/manifests/volsync.backube_replicationsources.yaml b/bundle/manifests/volsync.backube_replicationsources.yaml index c6177bc7a..752b57955 100644 --- a/bundle/manifests/volsync.backube_replicationsources.yaml +++ b/bundle/manifests/volsync.backube_replicationsources.yaml @@ -66,6 +66,10 @@ spec: provider. The name should be of the form: domain.com/provider.' type: string type: object + immediate: + description: immediate can be used to stop replication when a new + ReplicationSource is created. Defaults to "true". + type: boolean paused: description: paused can be used to temporarily stop replication. Defaults to "false". diff --git a/bundle/manifests/volsync.clusterserviceversion.yaml b/bundle/manifests/volsync.clusterserviceversion.yaml index 9362ddfd8..6d00d151f 100644 --- a/bundle/manifests/volsync.clusterserviceversion.yaml +++ b/bundle/manifests/volsync.clusterserviceversion.yaml @@ -55,7 +55,7 @@ metadata: } ] capabilities: Basic Install - createdAt: "2023-07-13T13:28:33Z" + createdAt: "2023-07-27T21:09:50Z" olm.skipRange: '>=0.4.0 <0.8.0' operators.operatorframework.io/builder: operator-sdk-v1.26.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 diff --git a/config/crd/bases/volsync.backube_replicationsources.yaml b/config/crd/bases/volsync.backube_replicationsources.yaml index 0af28b544..a1de6adae 100644 --- a/config/crd/bases/volsync.backube_replicationsources.yaml +++ b/config/crd/bases/volsync.backube_replicationsources.yaml @@ -67,6 +67,10 @@ spec: provider. The name should be of the form: domain.com/provider.' type: string type: object + immediate: + description: immediate can be used to stop replication when a new + ReplicationSource is created. Defaults to "true". + type: boolean paused: description: paused can be used to temporarily stop replication. Defaults to "false". diff --git a/controllers/mover/rclone/builder.go b/controllers/mover/rclone/builder.go index 82681785f..657b5a344 100644 --- a/controllers/mover/rclone/builder.go +++ b/controllers/mover/rclone/builder.go @@ -114,6 +114,8 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.Rclone.MoverServiceAccount) + isImmediate := source.Spec.Immediate == nil + return &Mover{ client: client, logger: logger.WithValues("method", "Rclone"), @@ -132,6 +134,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, privileged: privileged, moverSecurityContext: source.Spec.Rclone.MoverSecurityContext, latestMoverStatus: source.Status.LatestMoverStatus, + isImmediate: isImmediate, }, nil } diff --git a/controllers/mover/rclone/mover.go b/controllers/mover/rclone/mover.go index 3ed7a4d12..109bb2023 100644 --- a/controllers/mover/rclone/mover.go +++ b/controllers/mover/rclone/mover.go @@ -65,6 +65,7 @@ type Mover struct { privileged bool // true if the mover should have elevated privileges moverSecurityContext *corev1.PodSecurityContext latestMoverStatus *volsyncv1alpha1.MoverStatus + isImmediate bool } var _ mover.Mover = &Mover{} @@ -231,6 +232,9 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } + if !m.isImmediate && m.latestMoverStatus == nil { + parallelism = int32(0) + } job.Spec.Parallelism = ¶llelism envVars := []corev1.EnvVar{ diff --git a/controllers/mover/restic/builder.go b/controllers/mover/restic/builder.go index 73d18ed1f..e12c70eba 100644 --- a/controllers/mover/restic/builder.go +++ b/controllers/mover/restic/builder.go @@ -119,6 +119,8 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.Restic.MoverServiceAccount) + isImmediate := source.Spec.Immediate == nil + return &Mover{ client: client, logger: logger.WithValues("method", "Restic"), @@ -142,6 +144,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, unlock: source.Spec.Restic.Unlock, sourceStatus: source.Status.Restic, latestMoverStatus: source.Status.LatestMoverStatus, + isImmediate: isImmediate, }, nil } diff --git a/controllers/mover/restic/mover.go b/controllers/mover/restic/mover.go index ad5fb6b3d..0463f4530 100644 --- a/controllers/mover/restic/mover.go +++ b/controllers/mover/restic/mover.go @@ -79,6 +79,7 @@ type Mover struct { unlock string retainPolicy *volsyncv1alpha1.ResticRetainPolicy sourceStatus *volsyncv1alpha1.ReplicationSourceResticStatus + isImmediate bool // Destination-only fields previous *int32 restoreAsOf *string @@ -293,6 +294,9 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume if m.paused { parallelism = int32(0) } + if !m.isImmediate && m.latestMoverStatus == nil { + parallelism = int32(0) + } job.Spec.Parallelism = ¶llelism forgetOptions := generateForgetOptions(m.retainPolicy) // set default values diff --git a/controllers/mover/rsync/builder.go b/controllers/mover/rsync/builder.go index 0537ae8d9..6f74fdd83 100644 --- a/controllers/mover/rsync/builder.go +++ b/controllers/mover/rsync/builder.go @@ -119,6 +119,8 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, true, /*Rsync runs privileged only*/ source.Spec.Rsync.MoverServiceAccount) + isImmediate := source.Spec.Immediate == nil + return &Mover{ client: client, logger: logger.WithValues("method", "Rsync"), @@ -137,6 +139,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, mainPVCName: &source.Spec.SourcePVC, sourceStatus: source.Status.Rsync, latestMoverStatus: source.Status.LatestMoverStatus, + isImmediate: isImmediate, }, nil } diff --git a/controllers/mover/rsync/mover.go b/controllers/mover/rsync/mover.go index db9d4b39d..cf0fbc34e 100644 --- a/controllers/mover/rsync/mover.go +++ b/controllers/mover/rsync/mover.go @@ -65,6 +65,7 @@ type Mover struct { sourceStatus *volsyncv1alpha1.ReplicationSourceRsyncStatus destStatus *volsyncv1alpha1.ReplicationDestinationRsyncStatus latestMoverStatus *volsyncv1alpha1.MoverStatus + isImmediate bool } var _ mover.Mover = &Mover{} @@ -356,6 +357,9 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } + if !m.isImmediate && m.latestMoverStatus == nil { + parallelism = int32(0) + } job.Spec.Parallelism = ¶llelism readOnlyVolume := false diff --git a/controllers/mover/rsynctls/builder.go b/controllers/mover/rsynctls/builder.go index e97f2a645..4589e12c1 100644 --- a/controllers/mover/rsynctls/builder.go +++ b/controllers/mover/rsynctls/builder.go @@ -119,6 +119,8 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.RsyncTLS.MoverServiceAccount) + isImmediate := source.Spec.Immediate == nil + return &Mover{ client: client, logger: logger.WithValues("method", "RsyncTLS"), @@ -139,6 +141,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, moverSecurityContext: source.Spec.RsyncTLS.MoverSecurityContext, sourceStatus: source.Status.RsyncTLS, latestMoverStatus: source.Status.LatestMoverStatus, + isImmediate: isImmediate, }, nil } diff --git a/controllers/mover/rsynctls/mover.go b/controllers/mover/rsynctls/mover.go index e921fbd38..dd8e990b0 100644 --- a/controllers/mover/rsynctls/mover.go +++ b/controllers/mover/rsynctls/mover.go @@ -70,6 +70,7 @@ type Mover struct { sourceStatus *volsyncv1alpha1.ReplicationSourceRsyncTLSStatus destStatus *volsyncv1alpha1.ReplicationDestinationRsyncTLSStatus latestMoverStatus *volsyncv1alpha1.MoverStatus + isImmediate bool } var _ mover.Mover = &Mover{} @@ -363,6 +364,9 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } + if !m.isImmediate && m.latestMoverStatus == nil { + parallelism = int32(0) + } job.Spec.Parallelism = ¶llelism readOnlyVolume := false diff --git a/controllers/mover/syncthing/builder.go b/controllers/mover/syncthing/builder.go index aebeb5fc5..0bd8f06a7 100644 --- a/controllers/mover/syncthing/builder.go +++ b/controllers/mover/syncthing/builder.go @@ -135,6 +135,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, apiConfig: api.APIConfig{}, privileged: privileged, moverSecurityContext: source.Spec.Syncthing.MoverSecurityContext, + isImmediate: true, // defer setting the VolumeHandler }, nil } diff --git a/controllers/mover/syncthing/mover.go b/controllers/mover/syncthing/mover.go index 0c58bd2b9..649f73f91 100644 --- a/controllers/mover/syncthing/mover.go +++ b/controllers/mover/syncthing/mover.go @@ -119,6 +119,7 @@ type Mover struct { apiConfig api.APIConfig privileged bool moverSecurityContext *corev1.PodSecurityContext + isImmediate bool } var _ mover.Mover = &Mover{} diff --git a/helm/volsync/templates/volsync.backube_replicationsources.yaml b/helm/volsync/templates/volsync.backube_replicationsources.yaml index 78a2d9b4c..5c68b3d5f 100644 --- a/helm/volsync/templates/volsync.backube_replicationsources.yaml +++ b/helm/volsync/templates/volsync.backube_replicationsources.yaml @@ -60,6 +60,9 @@ spec: description: 'provider is the name of the external replication provider. The name should be of the form: domain.com/provider.' type: string type: object + immediate: + description: immediate can be used to stop replication when a new ReplicationSource is created. Defaults to "true". + type: boolean paused: description: paused can be used to temporarily stop replication. Defaults to "false". type: boolean From da6cadf28f6b66c86ed53c00e17e2dc3b78bf81e Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 27 Jul 2023 17:58:29 -0400 Subject: [PATCH 2/5] fix: check `latestMoverStatus.Result` instead of it being nil Signed-off-by: Devin Buhl --- controllers/mover/rclone/mover.go | 2 +- controllers/mover/restic/mover.go | 2 +- controllers/mover/rsync/mover.go | 2 +- controllers/mover/rsynctls/mover.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/mover/rclone/mover.go b/controllers/mover/rclone/mover.go index 109bb2023..b846af078 100644 --- a/controllers/mover/rclone/mover.go +++ b/controllers/mover/rclone/mover.go @@ -232,7 +232,7 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } - if !m.isImmediate && m.latestMoverStatus == nil { + if !m.isImmediate && m.latestMoverStatus.Result == "" { parallelism = int32(0) } job.Spec.Parallelism = ¶llelism diff --git a/controllers/mover/restic/mover.go b/controllers/mover/restic/mover.go index 0463f4530..8c5e53051 100644 --- a/controllers/mover/restic/mover.go +++ b/controllers/mover/restic/mover.go @@ -294,7 +294,7 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume if m.paused { parallelism = int32(0) } - if !m.isImmediate && m.latestMoverStatus == nil { + if !m.isImmediate && m.latestMoverStatus.Result == "" { parallelism = int32(0) } job.Spec.Parallelism = ¶llelism diff --git a/controllers/mover/rsync/mover.go b/controllers/mover/rsync/mover.go index cf0fbc34e..4ba0fbe4b 100644 --- a/controllers/mover/rsync/mover.go +++ b/controllers/mover/rsync/mover.go @@ -357,7 +357,7 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } - if !m.isImmediate && m.latestMoverStatus == nil { + if !m.isImmediate && m.latestMoverStatus.Result == "" { parallelism = int32(0) } job.Spec.Parallelism = ¶llelism diff --git a/controllers/mover/rsynctls/mover.go b/controllers/mover/rsynctls/mover.go index dd8e990b0..426dcafab 100644 --- a/controllers/mover/rsynctls/mover.go +++ b/controllers/mover/rsynctls/mover.go @@ -364,7 +364,7 @@ func (m *Mover) ensureJob(ctx context.Context, dataPVC *corev1.PersistentVolumeC if m.paused { parallelism = int32(0) } - if !m.isImmediate && m.latestMoverStatus == nil { + if !m.isImmediate && m.latestMoverStatus.Result == "" { parallelism = int32(0) } job.Spec.Parallelism = ¶llelism From 93c24cea10034ec9545eacf97546fd6266b515c4 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 27 Jul 2023 18:54:01 -0400 Subject: [PATCH 3/5] Update controllers/mover/restic/builder.go Co-authored-by: Thomas <9749173+uhthomas@users.noreply.github.com> Signed-off-by: Devin Buhl --- controllers/mover/restic/builder.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/controllers/mover/restic/builder.go b/controllers/mover/restic/builder.go index e12c70eba..f515591ea 100644 --- a/controllers/mover/restic/builder.go +++ b/controllers/mover/restic/builder.go @@ -119,7 +119,10 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.Restic.MoverServiceAccount) - isImmediate := source.Spec.Immediate == nil + isImmediate := true + if source.Spec.Immediate != nil { + isImmediate = *source.Spec.Immediate + } return &Mover{ client: client, From 03ff2a11a47015dc9ad05e8d7285f85fc78b28f2 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 27 Jul 2023 18:55:26 -0400 Subject: [PATCH 4/5] fix: update logic on Immediate Signed-off-by: Devin Buhl --- controllers/mover/rclone/builder.go | 5 ++++- controllers/mover/rsync/builder.go | 5 ++++- controllers/mover/rsynctls/builder.go | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/controllers/mover/rclone/builder.go b/controllers/mover/rclone/builder.go index 657b5a344..000c6b8fc 100644 --- a/controllers/mover/rclone/builder.go +++ b/controllers/mover/rclone/builder.go @@ -114,7 +114,10 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.Rclone.MoverServiceAccount) - isImmediate := source.Spec.Immediate == nil + isImmediate := true + if source.Spec.Immediate != nil { + isImmediate = *source.Spec.Immediate + } return &Mover{ client: client, diff --git a/controllers/mover/rsync/builder.go b/controllers/mover/rsync/builder.go index 6f74fdd83..f06d0acb8 100644 --- a/controllers/mover/rsync/builder.go +++ b/controllers/mover/rsync/builder.go @@ -119,7 +119,10 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, true, /*Rsync runs privileged only*/ source.Spec.Rsync.MoverServiceAccount) - isImmediate := source.Spec.Immediate == nil + isImmediate := true + if source.Spec.Immediate != nil { + isImmediate = *source.Spec.Immediate + } return &Mover{ client: client, diff --git a/controllers/mover/rsynctls/builder.go b/controllers/mover/rsynctls/builder.go index 4589e12c1..8a0153da6 100644 --- a/controllers/mover/rsynctls/builder.go +++ b/controllers/mover/rsynctls/builder.go @@ -119,7 +119,10 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger, saHandler := utils.NewSAHandler(client, source, isSource, privileged, source.Spec.RsyncTLS.MoverServiceAccount) - isImmediate := source.Spec.Immediate == nil + isImmediate := true + if source.Spec.Immediate != nil { + isImmediate = *source.Spec.Immediate + } return &Mover{ client: client, From f74f1f55cef22c09590692e8ab7d8f15e0385b88 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Thu, 27 Jul 2023 20:57:11 -0400 Subject: [PATCH 5/5] chore: fix linting Signed-off-by: Devin Buhl --- controllers/mover/rclone/builder.go | 1 + controllers/mover/restic/builder.go | 1 + controllers/mover/rsync/builder.go | 1 + controllers/mover/rsynctls/builder.go | 1 + 4 files changed, 4 insertions(+) diff --git a/controllers/mover/rclone/builder.go b/controllers/mover/rclone/builder.go index 000c6b8fc..a6e9be627 100644 --- a/controllers/mover/rclone/builder.go +++ b/controllers/mover/rclone/builder.go @@ -87,6 +87,7 @@ func (rb *Builder) getRcloneContainerImage() string { return rb.viper.GetString(rcloneContainerImageFlag) } +// nolint: funlen func (rb *Builder) FromSource(client client.Client, logger logr.Logger, eventRecorder events.EventRecorder, source *volsyncv1alpha1.ReplicationSource, privileged bool) (mover.Mover, error) { diff --git a/controllers/mover/restic/builder.go b/controllers/mover/restic/builder.go index f515591ea..34b291ef3 100644 --- a/controllers/mover/restic/builder.go +++ b/controllers/mover/restic/builder.go @@ -87,6 +87,7 @@ func (rb *Builder) getResticContainerImage() string { return rb.viper.GetString(resticContainerImageFlag) } +// nolint: funlen func (rb *Builder) FromSource(client client.Client, logger logr.Logger, eventRecorder events.EventRecorder, source *volsyncv1alpha1.ReplicationSource, privileged bool) (mover.Mover, error) { diff --git a/controllers/mover/rsync/builder.go b/controllers/mover/rsync/builder.go index f06d0acb8..3ba902423 100644 --- a/controllers/mover/rsync/builder.go +++ b/controllers/mover/rsync/builder.go @@ -87,6 +87,7 @@ func (rb *Builder) getRsyncContainerImage() string { return rb.viper.GetString(rsyncContainerImageFlag) } +// nolint: funlen func (rb *Builder) FromSource(client client.Client, logger logr.Logger, eventRecorder events.EventRecorder, source *volsyncv1alpha1.ReplicationSource, privileged bool) (mover.Mover, error) { diff --git a/controllers/mover/rsynctls/builder.go b/controllers/mover/rsynctls/builder.go index 8a0153da6..47d6eb2d1 100644 --- a/controllers/mover/rsynctls/builder.go +++ b/controllers/mover/rsynctls/builder.go @@ -87,6 +87,7 @@ func (rb *Builder) getRsyncTLSContainerImage() string { return rb.viper.GetString(rsyncTLSContainerImageFlag) } +// nolint: funlen func (rb *Builder) FromSource(client client.Client, logger logr.Logger, eventRecorder events.EventRecorder, source *volsyncv1alpha1.ReplicationSource, privileged bool) (mover.Mover, error) {