From 6764e1ade7de20ecffd88374d9b8e41972389c8d Mon Sep 17 00:00:00 2001 From: James Lu Date: Tue, 13 Aug 2024 20:27:55 +0800 Subject: [PATCH] fix(backup): customize the timeout of backup binary execution Add a default setting `backupExecutionTimeout` ref: longhorn/longhorn 8954,8319 Signed-off-by: James Lu --- controller/backup_target_controller.go | 13 +++++++++++- engineapi/backups.go | 29 ++++++++++++++++++-------- types/setting.go | 16 ++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go index b3f73af759..2f4ee536a5 100644 --- a/controller/backup_target_controller.go +++ b/controller/backup_target_controller.go @@ -24,6 +24,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" systembackupstore "github.com/longhorn/backupstore/systembackup" + lhtypes "github.com/longhorn/go-common-libs/types" "github.com/longhorn/longhorn-manager/datastore" "github.com/longhorn/longhorn-manager/engineapi" @@ -276,7 +277,17 @@ func newBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.Backu return nil, err } } - return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential), nil + + executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout) + if err != nil { + return nil, err + } + timeout := time.Duration(executeTimeout) * time.Minute + if executeTimeout <= 0 { + timeout = lhtypes.ExecuteNoTimeout + } + + return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil } func newBackupTargetClientFromDefaultEngineImage(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget) (*engineapi.BackupTargetClient, error) { diff --git a/engineapi/backups.go b/engineapi/backups.go index aab84934c3..3398d02c52 100644 --- a/engineapi/backups.go +++ b/engineapi/backups.go @@ -31,17 +31,19 @@ const ( ) type BackupTargetClient struct { - Image string - URL string - Credential map[string]string + Image string + URL string + Credential map[string]string + ExecuteTimeout time.Duration } // NewBackupTargetClient returns the backup target client -func NewBackupTargetClient(engineImage, url string, credential map[string]string) *BackupTargetClient { +func NewBackupTargetClient(engineImage, url string, credential map[string]string, executeTimeout time.Duration) *BackupTargetClient { return &BackupTargetClient{ - Image: engineImage, - URL: url, - Credential: credential, + Image: engineImage, + URL: url, + Credential: credential, + ExecuteTimeout: executeTimeout, } } @@ -68,7 +70,16 @@ func NewBackupTargetClientFromBackupTarget(backupTarget *longhorn.BackupTarget, } } - return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential), nil + executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout) + if err != nil { + return nil, err + } + timeout := time.Duration(executeTimeout) * time.Minute + if executeTimeout <= 0 { + timeout = lhtypes.ExecuteNoTimeout + } + + return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil } func (btc *BackupTargetClient) LonghornEngineBinary() string { @@ -130,7 +141,7 @@ func (btc *BackupTargetClient) ExecuteEngineBinary(args ...string) (string, erro if err != nil { return "", err } - return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, lhtypes.ExecuteDefaultTimeout) + return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, btc.ExecuteTimeout) } func (btc *BackupTargetClient) ExecuteEngineBinaryWithTimeout(timeout time.Duration, args ...string) (string, error) { diff --git a/types/setting.go b/types/setting.go index 0a15d00f2a..a2ac25096f 100644 --- a/types/setting.go +++ b/types/setting.go @@ -135,6 +135,7 @@ const ( SettingNameFreezeFilesystemForSnapshot = SettingName("freeze-filesystem-for-snapshot") SettingNameAutoCleanupSnapshotWhenDeleteBackup = SettingName("auto-cleanup-when-delete-backup") SettingNameDefaultMinNumberOfBackingImageCopies = SettingName("default-min-number-of-backing-image-copies") + SettingNameBackupExecutionTimeout = SettingName("backup-execution-timeout") SettingNameRWXVolumeFastFailover = SettingName("rwx-volume-fast-failover") ) @@ -227,6 +228,7 @@ var ( SettingNameFreezeFilesystemForSnapshot, SettingNameAutoCleanupSnapshotWhenDeleteBackup, SettingNameDefaultMinNumberOfBackingImageCopies, + SettingNameBackupExecutionTimeout, SettingNameRWXVolumeFastFailover, } ) @@ -347,6 +349,7 @@ var ( SettingNameFreezeFilesystemForSnapshot: SettingDefinitionFreezeFilesystemForSnapshot, SettingNameAutoCleanupSnapshotWhenDeleteBackup: SettingDefinitionAutoCleanupSnapshotWhenDeleteBackup, SettingNameDefaultMinNumberOfBackingImageCopies: SettingDefinitionDefaultMinNumberOfBackingImageCopies, + SettingNameBackupExecutionTimeout: SettingDefinitionBackupExecutionTimeout, SettingNameRWXVolumeFastFailover: SettingDefinitionRWXVolumeFastFailover, } @@ -409,6 +412,19 @@ var ( }, } + SettingDefinitionBackupExecutionTimeout = SettingDefinition{ + DisplayName: "Backup Execution Timeout", + Description: "In minutes. This setting determines the timeout for the backup engine execution. Set to 0 to disable the timeout.", + Category: SettingCategoryBackup, + Type: SettingTypeInt, + Required: true, + ReadOnly: false, + Default: "1", + ValueIntRange: map[string]int{ + ValueIntRangeMinimum: 0, + }, + } + SettingDefinitionRestoreVolumeRecurringJobs = SettingDefinition{ DisplayName: "Restore Volume Recurring Jobs", Description: "Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration.\n\n" +