From b5e5b7f97274fbf6f17192f19447d8618ab44448 Mon Sep 17 00:00:00 2001 From: Eric Weber Date: Fri, 24 May 2024 07:37:37 -0500 Subject: [PATCH] Add a global freeze setting and volume freeze field Longhorn 2187 Signed-off-by: Eric Weber --- api/model.go | 57 +++++++------ api/router.go | 13 +-- api/volume.go | 23 ++++++ client/README.md | 2 +- client/generated_client.go | 6 +- ...ated_update_freeze_fsfor_snapshot_input.go | 79 +++++++++++++++++++ client/generated_volume.go | 2 + .../backing_image_data_source_controller.go | 7 +- controller/setting_controller.go | 6 ++ controller/snapshot_controller.go | 7 +- controller/system_rollout_controller.go | 29 ++++--- controller/volume_controller.go | 7 +- csi/util.go | 10 ++- datastore/longhorn.go | 13 +++ engineapi/enginesim.go | 3 +- engineapi/proxy_snapshot.go | 5 +- engineapi/snapshot.go | 6 +- engineapi/types.go | 2 +- k8s/crds.yaml | 7 ++ k8s/pkg/apis/longhorn/v1beta2/volume.go | 12 +++ manager/engine.go | 7 +- manager/volume.go | 29 +++++++ types/setting.go | 13 +++ types/types.go | 9 +++ upgrade/v16xto170/upgrade.go | 4 + webhook/admission/admission.go | 2 +- webhook/resources/volume/mutator.go | 3 + 27 files changed, 306 insertions(+), 57 deletions(-) create mode 100644 client/generated_update_freeze_fsfor_snapshot_input.go diff --git a/api/model.go b/api/model.go index 776006af9a..c39ebb51a0 100644 --- a/api/model.go +++ b/api/model.go @@ -62,6 +62,7 @@ type Volume struct { OfflineReplicaRebuildingRequired bool `json:"offlineReplicaRebuildingRequired"` SnapshotMaxCount int `json:"snapshotMaxCount"` SnapshotMaxSize string `json:"snapshotMaxSize"` + FreezeFilesystemForSnapshot longhorn.FreezeFilesystemForSnapshot `json:"freezeFilesystemForSnapshot"` DiskSelector []string `json:"diskSelector"` NodeSelector []string `json:"nodeSelector"` @@ -371,6 +372,10 @@ type UpdateSnapshotMaxSize struct { SnapshotMaxSize string `json:"snapshotMaxSize"` } +type UpdateFreezeFilesystemForSnapshotInput struct { + FreezeFilesystemForSnapshot string `json:"freezeFilesystemForSnapshot"` +} + type PVCreateInput struct { PVName string `json:"pvName"` FSType string `json:"fsType"` @@ -629,6 +634,7 @@ func NewSchema() *client.Schemas { schemas.AddType("UpdateReplicaSoftAntiAffinityInput", UpdateReplicaSoftAntiAffinityInput{}) schemas.AddType("UpdateReplicaZoneSoftAntiAffinityInput", UpdateReplicaZoneSoftAntiAffinityInput{}) schemas.AddType("UpdateReplicaDiskSoftAntiAffinityInput", UpdateReplicaDiskSoftAntiAffinityInput{}) + schemas.AddType("UpdateFreezeFilesystemForSnapshotInput", UpdateFreezeFilesystemForSnapshotInput{}) schemas.AddType("workloadStatus", longhorn.WorkloadStatus{}) schemas.AddType("cloneStatus", longhorn.VolumeCloneStatus{}) schemas.AddType("empty", Empty{}) @@ -1031,6 +1037,10 @@ func volumeSchema(volume *client.Schema) { Input: "UpdateReplicaDiskSoftAntiAffinityInput", }, + "updateFreezeFilesystemForSnapshot": { + Input: "UpdateFreezeFilesystemForSnapshotInput", + }, + "pvCreate": { Input: "PVCreateInput", Output: "volume", @@ -1508,28 +1518,29 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor Actions: map[string]string{}, Links: map[string]string{}, }, - Name: v.Name, - Size: strconv.FormatInt(v.Spec.Size, 10), - Frontend: v.Spec.Frontend, - DisableFrontend: v.Spec.DisableFrontend, - LastAttachedBy: v.Spec.LastAttachedBy, - FromBackup: v.Spec.FromBackup, - DataSource: v.Spec.DataSource, - NumberOfReplicas: v.Spec.NumberOfReplicas, - ReplicaAutoBalance: v.Spec.ReplicaAutoBalance, - DataLocality: v.Spec.DataLocality, - SnapshotDataIntegrity: v.Spec.SnapshotDataIntegrity, - SnapshotMaxCount: v.Spec.SnapshotMaxCount, - SnapshotMaxSize: strconv.FormatInt(v.Spec.SnapshotMaxSize, 10), - BackupCompressionMethod: v.Spec.BackupCompressionMethod, - StaleReplicaTimeout: v.Spec.StaleReplicaTimeout, - Created: v.CreationTimestamp.String(), - Image: v.Spec.Image, - BackingImage: v.Spec.BackingImage, - Standby: v.Spec.Standby, - DiskSelector: v.Spec.DiskSelector, - NodeSelector: v.Spec.NodeSelector, - RestoreVolumeRecurringJob: v.Spec.RestoreVolumeRecurringJob, + Name: v.Name, + Size: strconv.FormatInt(v.Spec.Size, 10), + Frontend: v.Spec.Frontend, + DisableFrontend: v.Spec.DisableFrontend, + LastAttachedBy: v.Spec.LastAttachedBy, + FromBackup: v.Spec.FromBackup, + DataSource: v.Spec.DataSource, + NumberOfReplicas: v.Spec.NumberOfReplicas, + ReplicaAutoBalance: v.Spec.ReplicaAutoBalance, + DataLocality: v.Spec.DataLocality, + SnapshotDataIntegrity: v.Spec.SnapshotDataIntegrity, + SnapshotMaxCount: v.Spec.SnapshotMaxCount, + SnapshotMaxSize: strconv.FormatInt(v.Spec.SnapshotMaxSize, 10), + BackupCompressionMethod: v.Spec.BackupCompressionMethod, + StaleReplicaTimeout: v.Spec.StaleReplicaTimeout, + Created: v.CreationTimestamp.String(), + Image: v.Spec.Image, + BackingImage: v.Spec.BackingImage, + Standby: v.Spec.Standby, + DiskSelector: v.Spec.DiskSelector, + NodeSelector: v.Spec.NodeSelector, + RestoreVolumeRecurringJob: v.Spec.RestoreVolumeRecurringJob, + FreezeFilesystemForSnapshot: v.Spec.FreezeFilesystemForSnapshot, State: v.Status.State, Robustness: v.Status.Robustness, @@ -1607,6 +1618,7 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor actions["updateReplicaSoftAntiAffinity"] = struct{}{} actions["updateReplicaZoneSoftAntiAffinity"] = struct{}{} actions["updateReplicaDiskSoftAntiAffinity"] = struct{}{} + actions["updateFreezeFilesystemForSnapshot"] = struct{}{} actions["recurringJobAdd"] = struct{}{} actions["recurringJobDelete"] = struct{}{} actions["recurringJobList"] = struct{}{} @@ -1638,6 +1650,7 @@ func toVolumeResource(v *longhorn.Volume, ves []*longhorn.Engine, vrs []*longhor actions["updateReplicaSoftAntiAffinity"] = struct{}{} actions["updateReplicaZoneSoftAntiAffinity"] = struct{}{} actions["updateReplicaDiskSoftAntiAffinity"] = struct{}{} + actions["updateFreezeFilesystemForSnapshot"] = struct{}{} actions["pvCreate"] = struct{}{} actions["pvcCreate"] = struct{}{} actions["cancelExpansion"] = struct{}{} diff --git a/api/router.go b/api/router.go index 5e5a237b2b..1b36fafe29 100644 --- a/api/router.go +++ b/api/router.go @@ -84,12 +84,13 @@ func NewRouter(s *Server) *mux.Router { "expand": s.VolumeExpand, "cancelExpansion": s.VolumeCancelExpansion, - "updateReplicaCount": s.VolumeUpdateReplicaCount, - "updateReplicaAutoBalance": s.VolumeUpdateReplicaAutoBalance, - "updateSnapshotDataIntegrity": s.VolumeUpdateSnapshotDataIntegrity, - "updateBackupCompressionMethod": s.VolumeUpdateBackupCompressionMethod, - "updateOfflineReplicaRebuilding": s.VolumeUpdateOfflineReplicaRebuilding, - "replicaRemove": s.ReplicaRemove, + "updateReplicaCount": s.VolumeUpdateReplicaCount, + "updateReplicaAutoBalance": s.VolumeUpdateReplicaAutoBalance, + "updateSnapshotDataIntegrity": s.VolumeUpdateSnapshotDataIntegrity, + "updateBackupCompressionMethod": s.VolumeUpdateBackupCompressionMethod, + "updateOfflineReplicaRebuilding": s.VolumeUpdateOfflineReplicaRebuilding, + "updateFreezeFilesystemForSnapshot": s.VolumeUpdateFreezeFilesystemForSnapshot, + "replicaRemove": s.ReplicaRemove, "engineUpgrade": s.EngineUpgrade, diff --git a/api/volume.go b/api/volume.go index 3d2c6107aa..38cb10a993 100644 --- a/api/volume.go +++ b/api/volume.go @@ -200,6 +200,7 @@ func (s *Server) VolumeCreate(rw http.ResponseWriter, req *http.Request) error { ReplicaDiskSoftAntiAffinity: volume.ReplicaDiskSoftAntiAffinity, DataEngine: volume.DataEngine, OfflineReplicaRebuilding: volume.OfflineReplicaRebuilding, + FreezeFilesystemForSnapshot: volume.FreezeFilesystemForSnapshot, }, volume.RecurringJobSelector) if err != nil { return errors.Wrap(err, "failed to create volume") @@ -835,3 +836,25 @@ func (s *Server) VolumeUpdateSnapshotMaxSize(rw http.ResponseWriter, req *http.R } return s.responseWithVolume(rw, req, "", v) } + +func (s *Server) VolumeUpdateFreezeFilesystemForSnapshot(rw http.ResponseWriter, req *http.Request) error { + var input UpdateFreezeFilesystemForSnapshotInput + id := mux.Vars(req)["name"] + + apiContext := api.GetApiContext(req) + if err := apiContext.Read(&input); err != nil { + return errors.Wrap(err, "failed to read FreezeFilesystemForSnapshot input") + } + + obj, err := util.RetryOnConflictCause(func() (interface{}, error) { + return s.m.UpdateFreezeFilesystemForSnapshot(id, longhorn.FreezeFilesystemForSnapshot(input.FreezeFilesystemForSnapshot)) + }) + if err != nil { + return err + } + v, ok := obj.(*longhorn.Volume) + if !ok { + return fmt.Errorf("failed to convert to volume %v object", id) + } + return s.responseWithVolume(rw, req, "", v) +} diff --git a/client/README.md b/client/README.md index 8e0b5ba822..a84272dd17 100644 --- a/client/README.md +++ b/client/README.md @@ -32,7 +32,7 @@ GO111MODULE=off go get 5. Generate code. ```bash -scripts/generate-longhorn-schemas.sh http://: +GO111MODULE=off scripts/generate-longhorn-schemas.sh http://: ``` ## Copy code to longhorn-manager diff --git a/client/generated_client.go b/client/generated_client.go index 928dfbc5ff..4b9ce6155d 100644 --- a/client/generated_client.go +++ b/client/generated_client.go @@ -38,6 +38,7 @@ type RancherClient struct { UpdateReplicaSoftAntiAffinityInput UpdateReplicaSoftAntiAffinityInputOperations UpdateReplicaZoneSoftAntiAffinityInput UpdateReplicaZoneSoftAntiAffinityInputOperations UpdateReplicaDiskSoftAntiAffinityInput UpdateReplicaDiskSoftAntiAffinityInputOperations + UpdateFreezeFSForSnapshotInput UpdateFreezeFSForSnapshotInputOperations WorkloadStatus WorkloadStatusOperations CloneStatus CloneStatusOperations Empty EmptyOperations @@ -62,11 +63,11 @@ type RancherClient struct { Snapshot SnapshotOperations SnapshotCR SnapshotCROperations BackupVolume BackupVolumeOperations + BackupBackingImage BackupBackingImageOperations Setting SettingOperations RecurringJob RecurringJobOperations EngineImage EngineImageOperations BackingImage BackingImageOperations - BackupBackingImage BackupBackingImageOperations Node NodeOperations DiskUpdateInput DiskUpdateInputOperations DiskInfo DiskInfoOperations @@ -118,6 +119,7 @@ func constructClient(rancherBaseClient *RancherBaseClientImpl) *RancherClient { client.UpdateReplicaSoftAntiAffinityInput = newUpdateReplicaSoftAntiAffinityInputClient(client) client.UpdateReplicaZoneSoftAntiAffinityInput = newUpdateReplicaZoneSoftAntiAffinityInputClient(client) client.UpdateReplicaDiskSoftAntiAffinityInput = newUpdateReplicaDiskSoftAntiAffinityInputClient(client) + client.UpdateFreezeFSForSnapshotInput = newUpdateFreezeFSForSnapshotInputClient(client) client.WorkloadStatus = newWorkloadStatusClient(client) client.CloneStatus = newCloneStatusClient(client) client.Empty = newEmptyClient(client) @@ -142,6 +144,7 @@ func constructClient(rancherBaseClient *RancherBaseClientImpl) *RancherClient { client.Snapshot = newSnapshotClient(client) client.SnapshotCR = newSnapshotCRClient(client) client.BackupVolume = newBackupVolumeClient(client) + client.BackupBackingImage = newBackupBackingImageClient(client) client.Setting = newSettingClient(client) client.RecurringJob = newRecurringJobClient(client) client.EngineImage = newEngineImageClient(client) @@ -155,7 +158,6 @@ func constructClient(rancherBaseClient *RancherBaseClientImpl) *RancherClient { client.SystemBackup = newSystemBackupClient(client) client.SystemRestore = newSystemRestoreClient(client) client.SnapshotCRListOutput = newSnapshotCRListOutputClient(client) - client.BackupBackingImage = newBackupBackingImageClient(client) return client } diff --git a/client/generated_update_freeze_fsfor_snapshot_input.go b/client/generated_update_freeze_fsfor_snapshot_input.go new file mode 100644 index 0000000000..8773d570c5 --- /dev/null +++ b/client/generated_update_freeze_fsfor_snapshot_input.go @@ -0,0 +1,79 @@ +package client + +const ( + UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE = "UpdateFreezeFSForSnapshotInput" +) + +type UpdateFreezeFSForSnapshotInput struct { + Resource `yaml:"-"` + + FreezeFSForSnapshot string `json:"freezeFSForSnapshot,omitempty" yaml:"freeze_fsfor_snapshot,omitempty"` +} + +type UpdateFreezeFSForSnapshotInputCollection struct { + Collection + Data []UpdateFreezeFSForSnapshotInput `json:"data,omitempty"` + client *UpdateFreezeFSForSnapshotInputClient +} + +type UpdateFreezeFSForSnapshotInputClient struct { + rancherClient *RancherClient +} + +type UpdateFreezeFSForSnapshotInputOperations interface { + List(opts *ListOpts) (*UpdateFreezeFSForSnapshotInputCollection, error) + Create(opts *UpdateFreezeFSForSnapshotInput) (*UpdateFreezeFSForSnapshotInput, error) + Update(existing *UpdateFreezeFSForSnapshotInput, updates interface{}) (*UpdateFreezeFSForSnapshotInput, error) + ById(id string) (*UpdateFreezeFSForSnapshotInput, error) + Delete(container *UpdateFreezeFSForSnapshotInput) error +} + +func newUpdateFreezeFSForSnapshotInputClient(rancherClient *RancherClient) *UpdateFreezeFSForSnapshotInputClient { + return &UpdateFreezeFSForSnapshotInputClient{ + rancherClient: rancherClient, + } +} + +func (c *UpdateFreezeFSForSnapshotInputClient) Create(container *UpdateFreezeFSForSnapshotInput) (*UpdateFreezeFSForSnapshotInput, error) { + resp := &UpdateFreezeFSForSnapshotInput{} + err := c.rancherClient.doCreate(UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE, container, resp) + return resp, err +} + +func (c *UpdateFreezeFSForSnapshotInputClient) Update(existing *UpdateFreezeFSForSnapshotInput, updates interface{}) (*UpdateFreezeFSForSnapshotInput, error) { + resp := &UpdateFreezeFSForSnapshotInput{} + err := c.rancherClient.doUpdate(UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE, &existing.Resource, updates, resp) + return resp, err +} + +func (c *UpdateFreezeFSForSnapshotInputClient) List(opts *ListOpts) (*UpdateFreezeFSForSnapshotInputCollection, error) { + resp := &UpdateFreezeFSForSnapshotInputCollection{} + err := c.rancherClient.doList(UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE, opts, resp) + resp.client = c + return resp, err +} + +func (cc *UpdateFreezeFSForSnapshotInputCollection) Next() (*UpdateFreezeFSForSnapshotInputCollection, error) { + if cc != nil && cc.Pagination != nil && cc.Pagination.Next != "" { + resp := &UpdateFreezeFSForSnapshotInputCollection{} + err := cc.client.rancherClient.doNext(cc.Pagination.Next, resp) + resp.client = cc.client + return resp, err + } + return nil, nil +} + +func (c *UpdateFreezeFSForSnapshotInputClient) ById(id string) (*UpdateFreezeFSForSnapshotInput, error) { + resp := &UpdateFreezeFSForSnapshotInput{} + err := c.rancherClient.doById(UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE, id, resp) + if apiError, ok := err.(*ApiError); ok { + if apiError.StatusCode == 404 { + return nil, nil + } + } + return resp, err +} + +func (c *UpdateFreezeFSForSnapshotInputClient) Delete(container *UpdateFreezeFSForSnapshotInput) error { + return c.rancherClient.doResourceDelete(UPDATE_FREEZE_FSFOR_SNAPSHOT_INPUT_TYPE, &container.Resource) +} diff --git a/client/generated_volume.go b/client/generated_volume.go index d54221ae24..f717814213 100644 --- a/client/generated_volume.go +++ b/client/generated_volume.go @@ -37,6 +37,8 @@ type Volume struct { Encrypted bool `json:"encrypted,omitempty" yaml:"encrypted,omitempty"` + FreezeFilesystemForSnapshot string `json:"freezeFSForSnapshot,omitempty" yaml:"freeze_fsfor_snapshot,omitempty"` + FromBackup string `json:"fromBackup,omitempty" yaml:"from_backup,omitempty"` Frontend string `json:"frontend,omitempty" yaml:"frontend,omitempty"` diff --git a/controller/backing_image_data_source_controller.go b/controller/backing_image_data_source_controller.go index 2040a4c5a3..6c3a49bb95 100644 --- a/controller/backing_image_data_source_controller.go +++ b/controller/backing_image_data_source_controller.go @@ -836,6 +836,11 @@ func (c *BackingImageDataSourceController) prepareRunningParameters(bids *longho } } if newSnapshotRequired { + freezeFilesystem, err := c.ds.GetFreezeFilesystemForSnapshotSetting(e) + if err != nil { + return err + } + engineClientProxy, err := c.getEngineClientProxy(e) if err != nil { return err @@ -843,7 +848,7 @@ func (c *BackingImageDataSourceController) prepareRunningParameters(bids *longho defer engineClientProxy.Close() snapLabels := map[string]string{types.GetLonghornLabelKey(types.LonghornLabelSnapshotForExportingBackingImage): bids.Name} - snapshotName, err := engineClientProxy.SnapshotCreate(e, bids.Name+"-"+util.RandomID(), snapLabels) + snapshotName, err := engineClientProxy.SnapshotCreate(e, bids.Name+"-"+util.RandomID(), snapLabels, freezeFilesystem) if err != nil { return err } diff --git a/controller/setting_controller.go b/controller/setting_controller.go index a12b097292..49fc6d879d 100644 --- a/controller/setting_controller.go +++ b/controller/setting_controller.go @@ -1502,6 +1502,7 @@ const ( ClusterInfoVolumeRestoreVolumeRecurringJobCountFmt = "LonghornVolumeRestoreVolumeRecurringJob%sCount" ClusterInfoVolumeSnapshotDataIntegrityCountFmt = "LonghornVolumeSnapshotDataIntegrity%sCount" ClusterInfoVolumeUnmapMarkSnapChainRemovedCountFmt = "LonghornVolumeUnmapMarkSnapChainRemoved%sCount" + ClusterInfoVolumeFreezeFilesystemForSnapshotCountFmt = "LonghornVolumeFreezeFilesystemForSnapshot%sCount" ) // Node Scope Info: will be sent from all Longhorn cluster nodes @@ -1789,6 +1790,7 @@ func (info *ClusterInfo) collectVolumesInfo() error { restoreVolumeRecurringJobCountStruct := newStruct() snapshotDataIntegrityCountStruct := newStruct() unmapMarkSnapChainRemovedCountStruct := newStruct() + freezeFilesystemForSnapshotCountStruct := newStruct() for _, volume := range volumesRO { dataEngine := types.ValueUnknown if volume.Spec.DataEngine != "" { @@ -1845,6 +1847,9 @@ func (info *ClusterInfo) collectVolumesInfo() error { unmapMarkSnapChainRemoved := info.collectSettingInVolume(string(volume.Spec.UnmapMarkSnapChainRemoved), string(longhorn.UnmapMarkSnapChainRemovedIgnored), types.SettingNameRemoveSnapshotsDuringFilesystemTrim) unmapMarkSnapChainRemovedCountStruct[util.StructName(fmt.Sprintf(ClusterInfoVolumeUnmapMarkSnapChainRemovedCountFmt, util.ConvertToCamel(string(unmapMarkSnapChainRemoved), "-")))]++ + + freezeFilesystemForSnapshot := info.collectSettingInVolume(string(volume.Spec.FreezeFilesystemForSnapshot), string(longhorn.FreezeFilesystemForSnapshotDefault), types.SettingNameFreezeFilesystemForSnapshot) + freezeFilesystemForSnapshotCountStruct[util.StructName(fmt.Sprintf(ClusterInfoVolumeFreezeFilesystemForSnapshotCountFmt, util.ConvertToCamel(string(freezeFilesystemForSnapshot), "-")))]++ } info.structFields.fields.AppendCounted(accessModeCountStruct) info.structFields.fields.AppendCounted(dataEngineCountStruct) @@ -1858,6 +1863,7 @@ func (info *ClusterInfo) collectVolumesInfo() error { info.structFields.fields.AppendCounted(restoreVolumeRecurringJobCountStruct) info.structFields.fields.AppendCounted(snapshotDataIntegrityCountStruct) info.structFields.fields.AppendCounted(unmapMarkSnapChainRemovedCountStruct) + info.structFields.fields.AppendCounted(freezeFilesystemForSnapshotCountStruct) // TODO: Use the total volume count instead when v2 volume actual size is implemented. // https://github.com/longhorn/longhorn/issues/5947 diff --git a/controller/snapshot_controller.go b/controller/snapshot_controller.go index 18bd029479..cd027c6d27 100644 --- a/controller/snapshot_controller.go +++ b/controller/snapshot_controller.go @@ -628,6 +628,11 @@ func (sc *SnapshotController) getTheOnlyEngineCRforSnapshotRO(snapshot *longhorn } func (sc *SnapshotController) handleSnapshotCreate(snapshot *longhorn.Snapshot, engine *longhorn.Engine) error { + freezeFilesystem, err := sc.ds.GetFreezeFilesystemForSnapshotSetting(engine) + if err != nil { + return err + } + engineCliClient, err := GetBinaryClientForEngine(engine, sc.engineClientCollection, engine.Status.CurrentImage) if err != nil { return err @@ -645,7 +650,7 @@ func (sc *SnapshotController) handleSnapshotCreate(snapshot *longhorn.Snapshot, } if snapshotInfo == nil { sc.logger.Infof("Creating snapshot %v of volume %v", snapshot.Name, snapshot.Spec.Volume) - _, err = engineClientProxy.SnapshotCreate(engine, snapshot.Name, snapshot.Spec.Labels) + _, err = engineClientProxy.SnapshotCreate(engine, snapshot.Name, snapshot.Spec.Labels, freezeFilesystem) if err != nil { return err } diff --git a/controller/system_rollout_controller.go b/controller/system_rollout_controller.go index 248741b77d..ad6610f113 100644 --- a/controller/system_rollout_controller.go +++ b/controller/system_rollout_controller.go @@ -435,17 +435,19 @@ func (c *SystemRolloutController) systemRollout() error { wg := &sync.WaitGroup{} restoreFns := map[string]func() error{ - types.KubernetesKindServiceList: c.restoreService, - types.KubernetesKindServiceAccountList: c.restoreServiceAccounts, - types.KubernetesKindClusterRoleList: c.restoreClusterRoles, - types.KubernetesKindClusterRoleBindingList: c.restoreClusterRoleBindings, - types.KubernetesKindRoleList: c.restoreRoles, - types.KubernetesKindRoleBindingList: c.restoreRoleBindings, - types.KubernetesKindStorageClassList: c.restoreStorageClasses, - types.KubernetesKindConfigMapList: c.restoreConfigMaps, - types.KubernetesKindDeploymentList: c.restoreDeployments, - types.LonghornKindBackingImageList: c.restoreBackingIamges, - types.LonghornKindRecurringJobList: c.restoreRecurringJobs, + types.KubernetesKindServiceList: c.restoreService, + types.KubernetesKindServiceAccountList: c.restoreServiceAccounts, + types.KubernetesKindClusterRoleList: c.restoreClusterRoles, + types.KubernetesKindClusterRoleBindingList: c.restoreClusterRoleBindings, + types.KubernetesKindRoleList: c.restoreRoles, + types.KubernetesKindRoleBindingList: c.restoreRoleBindings, + types.KubernetesKindStorageClassList: c.restoreStorageClasses, + types.KubernetesKindConfigMapList: c.restoreConfigMaps, + types.KubernetesKindDeploymentList: c.restoreDeployments, + types.LonghornKindBackingImageList: c.restoreBackingIamges, + types.KubernetesKindPersistentVolumeList: c.restorePersistentVolumes, + types.KubernetesKindPersistentVolumeClaimList: c.restorePersistentVolumeClaims, + types.LonghornKindRecurringJobList: c.restoreRecurringJobs, } wg.Add(len(restoreFns)) for k, v := range restoreFns { @@ -457,11 +459,8 @@ func (c *SystemRolloutController) systemRollout() error { } wg.Wait() - // Need to wait until backing images are restored, so the volumes with backingimages won't be rejected by webhook when restoring. - // PV/PVC restoration depends on Volume, so move them after volume restoration. + // Need to wait until backingimages are restored, so the volumes with backingimages can be restored. c.restore(types.LonghornKindVolumeList, c.restoreVolumes, log) - c.restore(types.KubernetesKindPersistentVolumeList, c.restorePersistentVolumes, log) - c.restore(types.KubernetesKindPersistentVolumeClaimList, c.restorePersistentVolumeClaims, log) if len(c.cacheErrors) == 0 { c.updateSystemRolloutRecord(record, diff --git a/controller/volume_controller.go b/controller/volume_controller.go index 5c0cf261bb..1ee7667963 100644 --- a/controller/volume_controller.go +++ b/controller/volume_controller.go @@ -4410,6 +4410,11 @@ func (c *VolumeController) createSnapshot(snapshotName string, labels map[string return nil, err } + freezeFilesystem, err := c.ds.GetFreezeFilesystemForSnapshotSetting(e) + if err != nil { + return nil, err + } + engineCliClient, err := engineapi.GetEngineBinaryClient(c.ds, volume.Name, c.controllerID) if err != nil { return nil, err @@ -4434,7 +4439,7 @@ func (c *VolumeController) createSnapshot(snapshotName string, labels map[string } } - snapshotName, err = engineClientProxy.SnapshotCreate(e, snapshotName, labels) + snapshotName, err = engineClientProxy.SnapshotCreate(e, snapshotName, labels, freezeFilesystem) if err != nil { return nil, err } diff --git a/csi/util.go b/csi/util.go index 458a94a355..7d6527be09 100644 --- a/csi/util.go +++ b/csi/util.go @@ -196,7 +196,7 @@ func getVolumeOptions(volumeID string, volOptions map[string]string) (*longhornc if replicaDiskSoftAntiAffinity, ok := volOptions["replicaDiskSoftAntiAffinity"]; ok { if err := types.ValidateReplicaDiskSoftAntiAffinity(longhorn.ReplicaDiskSoftAntiAffinity(replicaDiskSoftAntiAffinity)); err != nil { - return nil, errors.Wrap(err, "Invalid parameter replicaDiskSoftAntiAffinity") + return nil, errors.Wrap(err, "invalid parameter replicaDiskSoftAntiAffinity") } vol.ReplicaDiskSoftAntiAffinity = replicaDiskSoftAntiAffinity } @@ -234,6 +234,14 @@ func getVolumeOptions(volumeID string, volOptions map[string]string) (*longhornc if driver, ok := volOptions["dataEngine"]; ok { vol.DataEngine = driver } + + if freezeFilesystemForSnapshot, ok := volOptions["freezeFilesystemForSnapshot"]; ok { + if err := types.ValidateFreezeFilesystemForSnapshot(longhorn.FreezeFilesystemForSnapshot(freezeFilesystemForSnapshot)); err != nil { + return nil, errors.Wrap(err, "invalid parameter freezeFilesystemForSnapshot") + } + vol.FreezeFilesystemForSnapshot = freezeFilesystemForSnapshot + } + return vol, nil } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index 042943bd72..54597f61c4 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -5111,3 +5111,16 @@ func (s *DataStore) GetRunningInstanceManagerByNodeRO(node string, dataEngine lo return nil, fmt.Errorf("failed to find a running instance manager for node %v", node) } + +func (s *DataStore) GetFreezeFilesystemForSnapshotSetting(e *longhorn.Engine) (bool, error) { + volume, err := s.GetVolumeRO(e.Spec.VolumeName) + if err != nil { + return false, err + } + + if volume.Spec.FreezeFilesystemForSnapshot != longhorn.FreezeFilesystemForSnapshotDefault { + return volume.Spec.FreezeFilesystemForSnapshot == longhorn.FreezeFilesystemForSnapshotEnabled, nil + } + + return s.GetSettingAsBool(types.SettingNameFreezeFilesystemForSnapshot) +} diff --git a/engineapi/enginesim.go b/engineapi/enginesim.go index bca0757d58..966b335fff 100644 --- a/engineapi/enginesim.go +++ b/engineapi/enginesim.go @@ -162,7 +162,8 @@ func (e *EngineSimulator) SimulateStopReplica(addr string) error { return nil } -func (e *EngineSimulator) SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string) (string, error) { +func (e *EngineSimulator) SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string, + freezeFilesystem bool) (string, error) { return "", fmt.Errorf(ErrNotImplement) } diff --git a/engineapi/proxy_snapshot.go b/engineapi/proxy_snapshot.go index f4a6e4db8f..57cfda02d0 100644 --- a/engineapi/proxy_snapshot.go +++ b/engineapi/proxy_snapshot.go @@ -4,9 +4,10 @@ import ( longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) -func (p *Proxy) SnapshotCreate(e *longhorn.Engine, name string, labels map[string]string) (string, error) { +func (p *Proxy) SnapshotCreate(e *longhorn.Engine, name string, labels map[string]string, + freezeFilesystem bool) (string, error) { return p.grpcClient.VolumeSnapshot(string(e.Spec.DataEngine), e.Name, e.Spec.VolumeName, p.DirectToURL(e), - name, labels) + name, labels, freezeFilesystem) } func (p *Proxy) SnapshotList(e *longhorn.Engine) (snapshots map[string]*longhorn.SnapshotInfo, err error) { diff --git a/engineapi/snapshot.go b/engineapi/snapshot.go index d9f3fd9d9c..eecb8871c8 100644 --- a/engineapi/snapshot.go +++ b/engineapi/snapshot.go @@ -15,13 +15,17 @@ import ( // SnapshotCreate calls engine binary // TODO: Deprecated, replaced by gRPC proxy -func (e *EngineBinary) SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string) (string, error) { +func (e *EngineBinary) SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string, + freezeFilesystem bool) (string, error) { args := []string{"snapshot", "create"} for k, v := range labels { args = append(args, "--label", k+"="+v) } args = append(args, name) + // Ignore freezeFilesystem argument. v1.7.0 engines understand the corresponding freeze-fs CLI flag, but older + // engines do not. + output, err := e.ExecuteEngineBinary(args...) if err != nil { return "", errors.Wrapf(err, "error creating snapshot '%s'", name) diff --git a/engineapi/types.go b/engineapi/types.go index 87822cb400..25cb1b2028 100644 --- a/engineapi/types.go +++ b/engineapi/types.go @@ -90,7 +90,7 @@ type EngineClient interface { ReplicaRebuildVerify(engine *longhorn.Engine, replicaName, url string) error ReplicaModeUpdate(engine *longhorn.Engine, url string, mode string) error - SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string) (string, error) + SnapshotCreate(engine *longhorn.Engine, name string, labels map[string]string, freezeFilesystem bool) (string, error) SnapshotList(engine *longhorn.Engine) (map[string]*longhorn.SnapshotInfo, error) SnapshotGet(engine *longhorn.Engine, name string) (*longhorn.SnapshotInfo, error) SnapshotDelete(engine *longhorn.Engine, name string) error diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 0ce71bd53c..3b16026549 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -3745,6 +3745,13 @@ spec: engineImage: description: 'Deprecated: Replaced by field `image`.' type: string + freezeFilesystemForSnapshot: + description: Setting that freezes the filesystem on the root partition before a snapshot is created. + enum: + - ignored + - enabled + - disabled + type: string fromBackup: type: string frontend: diff --git a/k8s/pkg/apis/longhorn/v1beta2/volume.go b/k8s/pkg/apis/longhorn/v1beta2/volume.go index de2a03c924..6f0da2487a 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/volume.go +++ b/k8s/pkg/apis/longhorn/v1beta2/volume.go @@ -160,6 +160,15 @@ const ( ReplicaDiskSoftAntiAffinityDisabled = ReplicaDiskSoftAntiAffinity("disabled") ) +// +kubebuilder:validation:Enum=ignored;enabled;disabled +type FreezeFilesystemForSnapshot string + +const ( + FreezeFilesystemForSnapshotDefault = FreezeFilesystemForSnapshot("ignored") + FreezeFilesystemForSnapshotEnabled = FreezeFilesystemForSnapshot("enabled") + FreezeFilesystemForSnapshotDisabled = FreezeFilesystemForSnapshot("disabled") +) + // Deprecated. type BackendStoreDriverType string @@ -298,6 +307,9 @@ type VolumeSpec struct { // +kubebuilder:validation:Type=string // +optional SnapshotMaxSize int64 `json:"snapshotMaxSize,string"` + // Setting that freezes the filesystem on the root partition before a snapshot is created. + // +optional + FreezeFilesystemForSnapshot FreezeFilesystemForSnapshot `json:"freezeFilesystemForSnapshot"` } // VolumeStatus defines the observed state of the Longhorn volume diff --git a/manager/engine.go b/manager/engine.go index bec839a6db..4d871d0e63 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -103,13 +103,18 @@ func (m *VolumeManager) CreateSnapshot(snapshotName string, labels map[string]st return nil, err } + freezeFilesystem, err := m.ds.GetFreezeFilesystemForSnapshotSetting(e) + if err != nil { + return nil, err + } + engineClientProxy, err := engineapi.GetCompatibleClient(e, engineCliClient, m.ds, nil, m.proxyConnCounter) if err != nil { return nil, err } defer engineClientProxy.Close() - snapshotName, err = engineClientProxy.SnapshotCreate(e, snapshotName, labels) + snapshotName, err = engineClientProxy.SnapshotCreate(e, snapshotName, labels, freezeFilesystem) if err != nil { return nil, err } diff --git a/manager/volume.go b/manager/volume.go index 511cad4984..4110287377 100644 --- a/manager/volume.go +++ b/manager/volume.go @@ -189,6 +189,7 @@ func (m *VolumeManager) Create(name string, spec *longhorn.VolumeSpec, recurring ReplicaDiskSoftAntiAffinity: spec.ReplicaDiskSoftAntiAffinity, DataEngine: spec.DataEngine, OfflineReplicaRebuilding: spec.OfflineReplicaRebuilding, + FreezeFilesystemForSnapshot: spec.FreezeFilesystemForSnapshot, }, } @@ -1231,3 +1232,31 @@ func (m *VolumeManager) restoreBackingImage(biName string) error { return nil } + +func (m *VolumeManager) UpdateFreezeFilesystemForSnapshot(name string, + freezeFilesystemForSnapshot longhorn.FreezeFilesystemForSnapshot) (v *longhorn.Volume, err error) { + defer func() { + err = errors.Wrapf(err, "unable to update field FreezeFilesystemForSnapshot for volume %v", name) + }() + + v, err = m.ds.GetVolume(name) + if err != nil { + return nil, err + } + + if v.Spec.FreezeFilesystemForSnapshot == freezeFilesystemForSnapshot { + logrus.Debugf("Volume %v already set field FreezeFilesystemForSnapshot to %v", v.Name, freezeFilesystemForSnapshot) + return v, nil + } + + oldFreezeFilesystemForSnapshot := v.Spec.FreezeFilesystemForSnapshot + v.Spec.FreezeFilesystemForSnapshot = freezeFilesystemForSnapshot + v, err = m.ds.UpdateVolume(v) + if err != nil { + return nil, err + } + + logrus.Infof("Updated volume %v field FreezeFilesystemForSnapshot from %v to %v", v.Name, + oldFreezeFilesystemForSnapshot, freezeFilesystemForSnapshot) + return v, nil +} diff --git a/types/setting.go b/types/setting.go index d043f39f78..ae282900e4 100644 --- a/types/setting.go +++ b/types/setting.go @@ -129,6 +129,7 @@ const ( SettingNameV2DataEngineGuaranteedInstanceManagerCPU = SettingName("v2-data-engine-guaranteed-instance-manager-cpu") SettingNameV2DataEngineLogLevel = SettingName("v2-data-engine-log-level") SettingNameV2DataEngineLogFlags = SettingName("v2-data-engine-log-flags") + SettingNameFreezeFilesystemForSnapshot = SettingName("freeze-filesystem-for-snapshot") ) var ( @@ -214,6 +215,7 @@ var ( SettingNameAllowEmptyNodeSelectorVolume, SettingNameAllowEmptyDiskSelectorVolume, SettingNameDisableSnapshotPurge, + SettingNameFreezeFilesystemForSnapshot, } ) @@ -327,6 +329,7 @@ var ( SettingNameAllowEmptyNodeSelectorVolume: SettingDefinitionAllowEmptyNodeSelectorVolume, SettingNameAllowEmptyDiskSelectorVolume: SettingDefinitionAllowEmptyDiskSelectorVolume, SettingNameDisableSnapshotPurge: SettingDefinitionDisableSnapshotPurge, + SettingNameFreezeFilesystemForSnapshot: SettingDefinitionFreezeFilesystemForSnapshot, } SettingDefinitionBackupTarget = SettingDefinition{ @@ -471,6 +474,16 @@ var ( Default: "false", } + SettingDefinitionFreezeFilesystemForSnapshot = SettingDefinition{ + DisplayName: "Freeze Filesystem For Snapshot", + Description: "Setting that freezes the filesystem on the root partition before a snapshot is created.", + Category: SettingCategorySnapshot, + Type: SettingTypeBool, + Required: true, + ReadOnly: false, + Default: "false", + } + SettingDefinitionReplicaAutoBalance = SettingDefinition{ DisplayName: "Replica Auto Balance", Description: "Enable this setting automatically rebalances replicas when discovered an available node.\n\n" + diff --git a/types/types.go b/types/types.go index deb9ccda58..4e38e5589d 100644 --- a/types/types.go +++ b/types/types.go @@ -872,6 +872,15 @@ func ValidateReplicaDiskSoftAntiAffinity(value longhorn.ReplicaDiskSoftAntiAffin return nil } +func ValidateFreezeFilesystemForSnapshot(value longhorn.FreezeFilesystemForSnapshot) error { + if value != longhorn.FreezeFilesystemForSnapshotDefault && + value != longhorn.FreezeFilesystemForSnapshotEnabled && + value != longhorn.FreezeFilesystemForSnapshotDisabled { + return fmt.Errorf("invalid FreezeFilesystemForSnapshot setting: %v", value) + } + return nil +} + func GetDaemonSetNameFromEngineImageName(engineImageName string) string { return "engine-image-" + engineImageName } diff --git a/upgrade/v16xto170/upgrade.go b/upgrade/v16xto170/upgrade.go index b2aaa08641..036698bbe3 100644 --- a/upgrade/v16xto170/upgrade.go +++ b/upgrade/v16xto170/upgrade.go @@ -56,6 +56,10 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM for _, v := range volumeMap { v.Spec.OfflineReplicaRebuilding = longhorn.OfflineReplicaRebuildingDisabled + + if v.Spec.FreezeFilesystemForSnapshot == "" { + v.Spec.FreezeFilesystemForSnapshot = longhorn.FreezeFilesystemForSnapshotDefault + } } return nil diff --git a/webhook/admission/admission.go b/webhook/admission/admission.go index 963851fed4..5b289de591 100644 --- a/webhook/admission/admission.go +++ b/webhook/admission/admission.go @@ -15,7 +15,7 @@ import ( ) const ( - AdmissionTypeValidation = "validation" + AdmissionTypeValidation = "validaton" AdmissionTypeMutation = "mutation" ) diff --git a/webhook/resources/volume/mutator.go b/webhook/resources/volume/mutator.go index d7ec37dd01..86004d569f 100644 --- a/webhook/resources/volume/mutator.go +++ b/webhook/resources/volume/mutator.go @@ -334,6 +334,9 @@ func mutate(newObj runtime.Object, moreLabels map[string]string) (admission.Patc if volume.Spec.DataLocality == longhorn.DataLocalityStrictLocal { patchOps = append(patchOps, `{"op": "replace", "path": "/spec/revisionCounterDisabled", "value": true}`) } + if string(volume.Spec.FreezeFilesystemForSnapshot) == "" { + patchOps = append(patchOps, fmt.Sprintf(`{"op": "replace", "path": "/spec/freezeFilesystemForSnapshot", "value": "%s"}`, longhorn.FreezeFilesystemForSnapshotDefault)) + } labels := volume.Labels if labels == nil {