diff --git a/api/model.go b/api/model.go index 05fc9f510c..b6f449a1e0 100644 --- a/api/model.go +++ b/api/model.go @@ -140,6 +140,7 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupCount string `json:"backupCount"` } type Backup struct { @@ -155,12 +156,15 @@ type Backup struct { Created string `json:"created"` Size string `json:"size"` Labels map[string]string `json:"labels"` + Parameters map[string]string `json:"parameters"` Messages map[string]string `json:"messages"` VolumeName string `json:"volumeName"` VolumeSize string `json:"volumeSize"` VolumeCreated string `json:"volumeCreated"` VolumeBackingImageName string `json:"volumeBackingImageName"` CompressionMethod string `json:"compressionMethod"` + NewlyUploadedDataSize string `json:"newlyUploadDataSize"` + ReUploadedDataSize string `json:"reUploadedDataSize"` } type BackupBackingImage struct { @@ -277,8 +281,9 @@ type DetachInput struct { } type SnapshotInput struct { - Name string `json:"name"` - Labels map[string]string `json:"labels"` + Name string `json:"name"` + Labels map[string]string `json:"labels"` + Parameters map[string]string `json:"parameters"` } type SnapshotCRInput struct { @@ -806,6 +811,11 @@ func recurringJobSchema(job *client.Schema) { labels.Type = "map[string]" labels.Nullable = true job.ResourceFields["labels"] = labels + + parameters := job.ResourceFields["parameters"] + parameters.Type = "map[string]" + parameters.Nullable = true + job.ResourceFields["parameters"] = parameters } func kubernetesStatusSchema(status *client.Schema) { @@ -1742,6 +1752,7 @@ func toBackupVolumeResource(bv *longhorn.BackupVolume, apiContext *api.ApiContex BackingImageName: bv.Status.BackingImageName, BackingImageChecksum: bv.Status.BackingImageChecksum, StorageClassName: bv.Status.StorageClassName, + BackupCount: bv.Status.BackupCount, } b.Actions = map[string]string{ "backupList": apiContext.UrlBuilder.ActionLink(b.Resource, "backupList"), @@ -1834,12 +1845,15 @@ func toBackupResource(b *longhorn.Backup) *Backup { Created: b.Status.BackupCreatedAt, Size: b.Status.Size, Labels: b.Status.Labels, + Parameters: b.Spec.Parameters, Messages: b.Status.Messages, VolumeName: b.Status.VolumeName, VolumeSize: b.Status.VolumeSize, VolumeCreated: b.Status.VolumeCreated, VolumeBackingImageName: b.Status.VolumeBackingImageName, CompressionMethod: string(b.Status.CompressionMethod), + NewlyUploadedDataSize: b.Status.NewlyUploadedDataSize, + ReUploadedDataSize: b.Status.ReUploadedDataSize, } // Set the volume name from backup CR's label if it's empty. // This field is empty probably because the backup state is not Ready @@ -2182,6 +2196,7 @@ func toRecurringJobResource(recurringJob *longhorn.RecurringJob, apiContext *api Retain: recurringJob.Spec.Retain, Concurrency: recurringJob.Spec.Concurrency, Labels: recurringJob.Spec.Labels, + Parameters: recurringJob.Spec.Parameters, }, } } diff --git a/api/recurringjob.go b/api/recurringjob.go index bb8a1f39c1..2a95f54590 100644 --- a/api/recurringjob.go +++ b/api/recurringjob.go @@ -63,6 +63,7 @@ func (s *Server) RecurringJobCreate(rw http.ResponseWriter, req *http.Request) e Retain: input.Retain, Concurrency: input.Concurrency, Labels: input.Labels, + Parameters: input.Parameters, }) if err != nil { return errors.Wrapf(err, "failed to create recurring job %v", input.Name) @@ -90,6 +91,7 @@ func (s *Server) RecurringJobUpdate(rw http.ResponseWriter, req *http.Request) e Retain: input.Retain, Concurrency: input.Concurrency, Labels: input.Labels, + Parameters: input.Parameters, }) }) if err != nil { diff --git a/api/snapshot.go b/api/snapshot.go index 05c55b23d3..e8247701f0 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -186,6 +186,14 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e return err } + backupParameters := map[string]string{} + if input.Parameters == nil { + backupParameters = input.Parameters + } + if err := util.ValidateBackupParameters(backupParameters); err != nil { + return err + } + // Cannot directly compare the structs since KubernetesStatus contains a slice which cannot be compared. if !reflect.DeepEqual(vol.Status.KubernetesStatus, longhorn.KubernetesStatus{}) { kubeStatus, err := json.Marshal(vol.Status.KubernetesStatus) @@ -195,7 +203,7 @@ func (s *Server) SnapshotBackup(w http.ResponseWriter, req *http.Request) (err e labels[types.KubernetesStatusLabel] = string(kubeStatus) } - if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels); err != nil { + if err := s.m.BackupSnapshot(bsutil.GenerateName("backup"), volName, input.Name, labels, backupParameters); err != nil { return err } diff --git a/app/recurring_job.go b/app/recurring_job.go index 6070a5598b..c569e40319 100644 --- a/app/recurring_job.go +++ b/app/recurring_job.go @@ -15,6 +15,7 @@ import ( "github.com/urfave/cli" "golang.org/x/sync/errgroup" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -31,6 +32,8 @@ import ( longhornclient "github.com/longhorn/longhorn-manager/client" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" + + lhbackup "github.com/longhorn/go-common-libs/backup" ) const ( @@ -56,6 +59,7 @@ type Job struct { retain int task longhorn.RecurringJobType labels map[string]string + parameters map[string]string eventRecorder record.EventRecorder @@ -121,6 +125,15 @@ func recurringJob(c *cli.Context) (err error) { return errors.Wrap(err, "failed to get JSON encoding for labels") } + jobParameterMap := map[string]string{} + if recurringJob.Spec.Parameters != nil { + jobParameterMap = recurringJob.Spec.Parameters + } + parameterJSON, err := json.Marshal(jobParameterMap) + if err != nil { + return errors.Wrap(err, "failed to get JSON encoding for parameters") + } + allowDetachedSetting := types.SettingNameAllowRecurringJobWhileVolumeDetached allowDetached, err := getSettingAsBoolean(allowDetachedSetting, namespace, lhClient) if err != nil { @@ -153,7 +166,7 @@ func recurringJob(c *cli.Context) (err error) { for _, volumeName := range filteredVolumes { startJobVolumeName := volumeName ewg.Go(func() error { - return startVolumeJob(startJobVolumeName, logger, concurrentLimiter, managerURL, jobName, jobTask, jobRetain, jobConcurrent, jobGroups, jobLabelMap, labelJSON) + return startVolumeJob(startJobVolumeName, logger, concurrentLimiter, managerURL, jobName, jobTask, jobRetain, jobConcurrent, jobGroups, jobLabelMap, labelJSON, jobParameterMap, parameterJSON) }) } @@ -162,7 +175,7 @@ func recurringJob(c *cli.Context) (err error) { func startVolumeJob( volumeName string, logger *logrus.Logger, concurrentLimiter chan struct{}, managerURL string, - jobName string, jobTask longhorn.RecurringJobType, jobRetain int, jobConcurrent int, jobGroups []string, jobLabelMap map[string]string, labelJSON []byte) error { + jobName string, jobTask longhorn.RecurringJobType, jobRetain int, jobConcurrent int, jobGroups []string, jobLabelMap map[string]string, labelJSON []byte, jobParameterMap map[string]string, parameterJSON []byte) error { concurrentLimiter <- struct{}{} defer func() { @@ -177,6 +190,7 @@ func startVolumeJob( "concurrent": jobConcurrent, "groups": strings.Join(jobGroups, ","), "labels": string(labelJSON), + "parameters": string(parameterJSON), }) log.Info("Creating job") @@ -187,6 +201,7 @@ func startVolumeJob( volumeName, snapshotName, jobLabelMap, + jobParameterMap, jobRetain, jobTask) if err != nil { @@ -213,7 +228,7 @@ func sliceStringSafely(s string, begin, end int) string { return s[begin:end] } -func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName string, labels map[string]string, retain int, task longhorn.RecurringJobType) (*Job, error) { +func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName string, labels map[string]string, parameters map[string]string, retain int, task longhorn.RecurringJobType) (*Job, error) { namespace := os.Getenv(types.EnvPodNamespace) if namespace == "" { return nil, fmt.Errorf("failed detect pod namespace, environment variable %v is missing", types.EnvPodNamespace) @@ -263,6 +278,7 @@ func newJob(logger logrus.FieldLogger, managerURL, volumeName, snapshotName stri volumeName: volumeName, snapshotName: snapshotName, labels: labels, + parameters: parameters, retain: retain, task: task, api: apiClient, @@ -611,9 +627,39 @@ func (job *Job) doRecurringBackup() (err error) { return err } + backupParameters := map[string]string{} + if intervalStr, exists := job.parameters[types.RecurringJobBackupParameterFullBackupInterval]; exists { + interval, err := strconv.Atoi(intervalStr) + if err != nil { + return errors.Wrapf(err, "interval %v is not number", intervalStr) + } + + if interval != 0 { + backupVolume, err := job.api.BackupVolume.ById(job.volumeName) + if err != nil { + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "failed to get backup volume %v", job.volumeName) + } + } + + backupCount := 0 + if backupVolume != nil && backupVolume.BackupCount != "" { + backupCount, err = strconv.Atoi(backupVolume.BackupCount) + if err != nil { + return errors.Wrapf(err, "backup count %v is not number", backupVolume.BackupCount) + } + } + + if backupCount%interval == 0 { + backupParameters[lhbackup.LonghornBackupParameterBackupMode] = lhbackup.LonghornBackupModeFull + } + } + } + if _, err := job.api.Volume.ActionSnapshotBackup(volume, &longhornclient.SnapshotInput{ - Labels: job.labels, - Name: job.snapshotName, + Labels: job.labels, + Name: job.snapshotName, + Parameters: job.parameters, }); err != nil { return err } diff --git a/client/generated_backup_volume.go b/client/generated_backup_volume.go index b1936e1988..19b1299cca 100644 --- a/client/generated_backup_volume.go +++ b/client/generated_backup_volume.go @@ -11,6 +11,8 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName,omitempty" yaml:"backing_image_name,omitempty"` + BackupCount string `json:"backupCount,omitempty" yaml:"backup_count,omitempty"` + Created string `json:"created,omitempty" yaml:"created,omitempty"` DataStored string `json:"dataStored,omitempty" yaml:"data_stored,omitempty"` diff --git a/client/generated_snapshot_input.go b/client/generated_snapshot_input.go index f8b6c78aee..b59a38f1a2 100644 --- a/client/generated_snapshot_input.go +++ b/client/generated_snapshot_input.go @@ -9,6 +9,8 @@ type SnapshotInput struct { Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` + Parameters map[string]string `json:"parameters,omitempty" yaml:"parameters,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` } diff --git a/controller/backup_controller.go b/controller/backup_controller.go index e4e830a577..819edad24b 100644 --- a/controller/backup_controller.go +++ b/controller/backup_controller.go @@ -448,6 +448,8 @@ func (bc *BackupController) reconcile(backupName string) (err error) { backup.Status.VolumeBackingImageName = backupInfo.VolumeBackingImageName backup.Status.CompressionMethod = longhorn.BackupCompressionMethod(backupInfo.CompressionMethod) backup.Status.LastSyncedAt = syncTime + backup.Status.NewlyUploadedDataSize = backupInfo.NewlyUploadedDataSize + backup.Status.ReUploadedDataSize = backupInfo.ReUploadedDataSize return nil } diff --git a/controller/backup_volume_controller.go b/controller/backup_volume_controller.go index f018d7d477..5d7cc518a9 100644 --- a/controller/backup_volume_controller.go +++ b/controller/backup_volume_controller.go @@ -383,6 +383,8 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error return nil } + logrus.Infof("[DEBUG]: backupVolumeInfo: %v", backupVolumeInfo) + // Update the Backup CR spec.syncRequestAt to request the // backup_controller to reconcile the Backup CR if the last backup changed if backupVolume.Status.LastBackupName != backupVolumeInfo.LastBackupName { @@ -408,6 +410,7 @@ func (bvc *BackupVolumeController) reconcile(backupVolumeName string) (err error backupVolume.Status.BackingImageChecksum = backupVolumeInfo.BackingImageChecksum backupVolume.Status.StorageClassName = backupVolumeInfo.StorageClassName backupVolume.Status.LastSyncedAt = syncTime + backupVolume.Status.BackupCount = backupVolumeInfo.BackupCount return nil } diff --git a/datastore/longhorn.go b/datastore/longhorn.go index a337588440..2cc1464286 100644 --- a/datastore/longhorn.go +++ b/datastore/longhorn.go @@ -4139,6 +4139,41 @@ func ValidateRecurringJob(job longhorn.RecurringJobSpec) error { return err } } + if job.Parameters != nil { + if err := ValidateRecurringJobParameters(job.Task, job.Labels); err != nil { + return err + } + } + return nil +} + +func ValidateRecurringJobParameters(task longhorn.RecurringJobType, parameters map[string]string) (err error) { + switch task { + case longhorn.RecurringJobTypeBackup, longhorn.RecurringJobTypeBackupForceCreate: + for key, value := range parameters { + if err := validateRecurringJobBackupParameter(key, value); err != nil { + return errors.Wrapf(err, "failed to validate recurring job backup task parameters") + } + } + // we don't support any parameters for other tasks currently + default: + return nil + } + + return nil +} + +func validateRecurringJobBackupParameter(key, value string) error { + switch key { + case types.RecurringJobBackupParameterFullBackupInterval: + _, err := strconv.Atoi(value) + if err != nil { + return errors.Wrapf(err, "%v:%v is not number", key, value) + } + default: + return fmt.Errorf("%v:%v is not a valid parameter", key, value) + } + return nil } diff --git a/engineapi/backup_monitor.go b/engineapi/backup_monitor.go index 17ab32afc9..74602fdc35 100644 --- a/engineapi/backup_monitor.go +++ b/engineapi/backup_monitor.go @@ -84,7 +84,7 @@ func NewBackupMonitor(logger logrus.FieldLogger, ds *datastore.DataStore, backup } _, replicaAddress, err := engineClientProxy.SnapshotBackup(engine, backup.Spec.SnapshotName, backup.Name, backupTargetClient.URL, volume.Spec.BackingImage, biChecksum, string(compressionMethod), concurrentLimit, storageClassName, - backup.Spec.Labels, backupTargetClient.Credential) + backup.Spec.Labels, backupTargetClient.Credential, backup.Spec.Parameters) if err != nil { if !strings.Contains(err.Error(), "DeadlineExceeded") { m.logger.WithError(err).Warn("Cannot take snapshot backup") diff --git a/engineapi/backups.go b/engineapi/backups.go index 10d21929b6..13aa7acc73 100644 --- a/engineapi/backups.go +++ b/engineapi/backups.go @@ -232,6 +232,7 @@ func (btc *BackupTargetClient) BackupVolumeDelete(destURL, volumeName string, cr // parseBackupVolumeConfig parses a backup volume config func parseBackupVolumeConfig(output string) (*BackupVolume, error) { backupVolume := new(BackupVolume) + logrus.Infof("[DEBUG] parseBackupVolumeConfig output: %v", output) if err := json.Unmarshal([]byte(output), backupVolume); err != nil { return nil, errors.Wrapf(err, "error parsing one backup volume config: \n%s", output) } @@ -318,7 +319,7 @@ func (btc *BackupTargetClient) BackupCleanUpAllMounts() (err error) { // TODO: Deprecated, replaced by gRPC proxy func (e *EngineBinary) SnapshotBackup(engine *longhorn.Engine, snapName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { if snapName == etypes.VolumeHeadName { return "", "", fmt.Errorf("invalid operation: cannot backup %v", etypes.VolumeHeadName) } diff --git a/engineapi/enginesim.go b/engineapi/enginesim.go index bca0757d58..2763913295 100644 --- a/engineapi/enginesim.go +++ b/engineapi/enginesim.go @@ -192,7 +192,7 @@ func (e *EngineSimulator) SnapshotPurgeStatus(*longhorn.Engine) (map[string]*lon func (e *EngineSimulator) SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { return "", "", fmt.Errorf(ErrNotImplement) } diff --git a/engineapi/proxy_backup.go b/engineapi/proxy_backup.go index 39c05f7d0b..21a10c6e9d 100644 --- a/engineapi/proxy_backup.go +++ b/engineapi/proxy_backup.go @@ -14,7 +14,7 @@ import ( func (p *Proxy) SnapshotBackup(e *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, - labels, credential map[string]string) (string, string, error) { + labels, credential, parameters map[string]string) (string, string, error) { if snapshotName == etypes.VolumeHeadName { return "", "", fmt.Errorf("invalid operation: cannot backup %v", etypes.VolumeHeadName) } @@ -40,7 +40,7 @@ func (p *Proxy) SnapshotBackup(e *longhorn.Engine, snapshotName, backupName, bac backupID, replicaAddress, err := p.grpcClient.SnapshotBackup(string(e.Spec.DataEngine), e.Name, e.Spec.VolumeName, p.DirectToURL(e), backupName, snapshotName, backupTarget, backingImageName, - backingImageChecksum, compressionMethod, concurrentLimit, storageClassName, labels, credentialEnv, + backingImageChecksum, compressionMethod, concurrentLimit, storageClassName, labels, credentialEnv, parameters, ) if err != nil { return "", "", err diff --git a/engineapi/types.go b/engineapi/types.go index 63d6be8a4f..9cefcd7f55 100644 --- a/engineapi/types.go +++ b/engineapi/types.go @@ -98,7 +98,7 @@ type EngineClient interface { SnapshotRevert(engine *longhorn.Engine, name string) error SnapshotPurge(engine *longhorn.Engine) error SnapshotPurgeStatus(engine *longhorn.Engine) (map[string]*longhorn.PurgeStatus, error) - SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, labels, credential map[string]string) (string, string, error) + SnapshotBackup(engine *longhorn.Engine, snapshotName, backupName, backupTarget, backingImageName, backingImageChecksum, compressionMethod string, concurrentLimit int, storageClassName string, labels, credential, parameters map[string]string) (string, string, error) SnapshotBackupStatus(engine *longhorn.Engine, backupName, replicaAddress, replicaName string) (*longhorn.EngineBackupStatus, error) SnapshotCloneStatus(engine *longhorn.Engine) (map[string]*longhorn.SnapshotCloneStatus, error) SnapshotClone(engine *longhorn.Engine, snapshotName, fromEngineAddress, fromVolumeName, fromEngineName string, fileSyncHTTPClientTimeout int64) error @@ -162,6 +162,7 @@ type BackupVolume struct { BackingImageName string `json:"backingImageName"` BackingImageChecksum string `json:"backingImageChecksum"` StorageClassName string `json:"storageClassName"` + BackupCount string `json:"backupCount"` } type Backup struct { @@ -179,6 +180,9 @@ type Backup struct { VolumeBackingImageName string `json:"volumeBackingImageName"` Messages map[string]string `json:"messages"` CompressionMethod string `json:"compressionMethod"` + Parameters map[string]string `json:"parameters"` + NewlyUploadedDataSize string `json:"newlyUploadedDataSize"` + ReUploadedDataSize string `json:"reUploadedDataSize"` } type ConfigMetadata struct { diff --git a/go.mod b/go.mod index 00cf3c5ed5..f2f86e4eb4 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,14 @@ module github.com/longhorn/longhorn-manager go 1.21 +replace github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 => github.com/ChanYiLin/backupstore v0.0.0-20240401081421-d2e974ac7664 + +replace github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc => github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305 + +replace github.com/longhorn/longhorn-engine v1.6.0-dev-20240105.0.20240126141003-067f67803ee8 => github.com/ChanYiLin/longhorn-engine v1.4.0-rc1.0.20240401081604-25b91ed6dab2 + +replace github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240308123529-28ec1b59c683 => github.com/ChanYiLin/longhorn-instance-manager v1.4.0-rc1.0.20240402045006-de7e77b6146a + replace ( k8s.io/api => k8s.io/api v0.28.5 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.5 @@ -83,13 +91,13 @@ require ( require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/benbjohnson/clock v1.3.0 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/mitchellh/go-ps v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect @@ -97,34 +105,34 @@ require ( golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 // indirect golang.org/x/tools v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/RoaringBitmap/roaring v1.9.0 // indirect + github.com/RoaringBitmap/roaring v1.9.1 // indirect github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/avast/retry-go v3.0.0+incompatible github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.12.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gammazero/deque v0.2.1 // indirect github.com/gammazero/workerpool v1.1.3 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -145,25 +153,25 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect - github.com/longhorn/longhorn-spdk-engine v0.0.0-20240308061103-33861f26f36e // indirect + github.com/longhorn/longhorn-spdk-engine v0.0.0-20240326225129-c0ac4cbe90cb // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/sys/mountinfo v0.7.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/selinux v1.10.0 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.47.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rancher/lasso v0.0.0-20240123150939-7055397d6dfa github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/slok/goresilience v0.2.0 // indirect github.com/spf13/cobra v1.7.0 // indirect @@ -186,7 +194,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.6.0 golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 @@ -201,7 +209,7 @@ require ( k8s.io/component-helpers v0.28.5 // indirect k8s.io/controller-manager v0.28.5 // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kms v0.28.5 // indirect k8s.io/kube-aggregator v0.25.4 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect diff --git a/go.sum b/go.sum index b3f91f24d3..59edf71767 100644 --- a/go.sum +++ b/go.sum @@ -605,6 +605,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ChanYiLin/backupstore v0.0.0-20240401081421-d2e974ac7664 h1:qvtjWMa4x7oqAhu4oSwA++np6RJt9zoGBk2GRCZedc4= +github.com/ChanYiLin/backupstore v0.0.0-20240401081421-d2e974ac7664/go.mod h1:S9DXMzVYentzBto6WVfNqhYKMy8t0Xuqx0Mxg1v3nNs= +github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305 h1:6myS9bHkMPBZkdU21ijovlf2GGWxdOZN8aWu1+RzPfE= +github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305/go.mod h1:ESTw7LYBF+dB5VndQNKXKrD6B9s/hF94lotGKXLovlM= +github.com/ChanYiLin/longhorn-instance-manager v1.4.0-rc1.0.20240402045006-de7e77b6146a h1:XBGDi7bAq+L98m1nM09b8NExi26ufUDZnPlXtxVKVWA= +github.com/ChanYiLin/longhorn-instance-manager v1.4.0-rc1.0.20240402045006-de7e77b6146a/go.mod h1:pLcWCGOTDE7C+S4abF74apK7rb7aUOruzpGmcoBH+DE= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -612,8 +618,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE= -github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I= +github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -649,8 +655,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= @@ -705,8 +712,9 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -749,8 +757,9 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -790,15 +799,16 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= @@ -1045,22 +1055,16 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/longhorn/backing-image-manager v1.6.0 h1:Jmlc8+W63l0VZoVhPwNLniAk+eBC4CNaadoqpqA51KE= github.com/longhorn/backing-image-manager v1.6.0/go.mod h1:IH0mgbK+Dr13xkY+LhDaufyd9YIpiKqYo1AeRLFYGrk= -github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 h1:iJRq59kA22f9HIjFtY/lz5rKCorZJrrYXju70XoWdmE= -github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k= -github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc h1:Eh9Npc5yBcVD8E4zVQIGUtC62HcfqevrHjQ2kh7fJ/E= -github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc/go.mod h1:ESTw7LYBF+dB5VndQNKXKrD6B9s/hF94lotGKXLovlM= github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425 h1:koSD52H0VkzJAh3OIZCdgQ9mqoRXklkeuhqmuwQ1WzU= github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425/go.mod h1:2aM6KBix3Khd56I4rihOBOOPOm0/YvYMjtr1KNclQsI= github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9 h1:spgyLW1ND0GZJkzFyfJvbvcjVtAwBU2Msg+Jy2Yy02k= github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9/go.mod h1:WGm84AyXymx7L0CqOw8KEr9okywD+Cj5xZi+eKeOoiU= github.com/longhorn/longhorn-engine v1.6.0 h1:6CH2vvwCgFBIGW4TegcI79CL1Ego1nvLZIC3ioRjjdM= github.com/longhorn/longhorn-engine v1.6.0/go.mod h1:Snkv3gy4AUOhZSYMI7g7lVX/OOB8DTo28eJwsEfbAwM= -github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240308123529-28ec1b59c683 h1:v7JKvBbdKKsqGV96oDY7eTOodbX8bLNyPlcOnI+vanM= -github.com/longhorn/longhorn-instance-manager v1.7.0-dev.0.20240308123529-28ec1b59c683/go.mod h1:paydDjcI9whUJd09/4QJQVDV9MSJDwXUD1872vhnREo= github.com/longhorn/longhorn-share-manager v1.6.0 h1:rmvwCWOeADdf67uw5Dsv7m5m0aFbIxERNk0rYl3DuVk= github.com/longhorn/longhorn-share-manager v1.6.0/go.mod h1:akAmB/ZKonalweOYEhbg85X2vhAz+/sMzg779EFgPO8= -github.com/longhorn/longhorn-spdk-engine v0.0.0-20240308061103-33861f26f36e h1:OPtYaSleQOlf+VsCF3hzS96nYj1F+X0dwky+LvsLh1M= -github.com/longhorn/longhorn-spdk-engine v0.0.0-20240308061103-33861f26f36e/go.mod h1:mgXCx3ZJpaObXmOUgczwzuYySs5Wc7SQK6sJtIRR8Ho= +github.com/longhorn/longhorn-spdk-engine v0.0.0-20240326225129-c0ac4cbe90cb h1:3n20Ufdz5UXCv7N2WMp0pXAmlpTxk1V3KRDRO8DDCT0= +github.com/longhorn/longhorn-spdk-engine v0.0.0-20240326225129-c0ac4cbe90cb/go.mod h1:XX36oZpuOnXWo/Qjyrd0oXBZdFN4sqz2S68A6pXnNNI= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= @@ -1093,8 +1097,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= +github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -1159,8 +1163,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1172,8 +1176,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= @@ -1241,8 +1246,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1275,8 +1280,9 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= +github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -1599,8 +1605,8 @@ golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2075,8 +2081,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa h1:RBgMaUMP+6soRkik4VoN8ojR2nex2TqZwjSSogic+eo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2211,8 +2217,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.28.5 h1:1oOZBdleMSRu14dT+EH2wnnBNsN0XfUnQx0QVan1xwc= k8s.io/kms v0.28.5/go.mod h1:BuOJ8gZRpTFPw4g/OXNfy9ljhuG+U2mT8SK3+3TuCxs= k8s.io/kube-aggregator v0.28.5 h1:gUe597PoEXlBjlKtHoce6T6W5F38IG2WlYMqwBqpwvE= diff --git a/k8s/crds.yaml b/k8s/crds.yaml index 64851fe1bb..7227a63004 100644 --- a/k8s/crds.yaml +++ b/k8s/crds.yaml @@ -739,6 +739,11 @@ spec: type: string description: The labels of snapshot backup. type: object + parameters: + additionalProperties: + type: string + description: The parameters of snapshot backup. + type: object snapshotName: description: The snapshot name. type: string @@ -777,12 +782,18 @@ spec: description: The error messages when calling longhorn engine on listing or inspecting backups. nullable: true type: object + newlyUploadDataSize: + description: Size of newly uploaded data + type: string ownerID: description: The node ID on which the controller is responsible to reconcile this backup CR. type: string progress: description: The snapshot backup progress. type: integer + reUploadedDataSize: + description: Size of reuploaded data + type: string replicaAddress: description: The address of the replica that runs snapshot backup. type: string @@ -831,22 +842,11 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null labels: longhorn-manager: "" name: backuptargets.longhorn.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: longhorn-conversion-webhook - namespace: longhorn-system - path: /v1/webhook/conversion - port: 9443 - conversionReviewVersions: - - v1beta2 - - v1beta1 group: longhorn.io names: kind: BackupTarget @@ -1111,6 +1111,9 @@ spec: backingImageName: description: The backing image name. type: string + backupCount: + description: The number of backups that have been created. + type: string createdAt: description: The backup volume creation time. type: string @@ -2476,6 +2479,11 @@ spec: name: description: The recurring job name. type: string + parameters: + additionalProperties: + type: string + description: The parameters of the snapshot/backup. + type: object retain: description: The retain count of the snapshot/backup. type: integer diff --git a/k8s/generate_code.sh b/k8s/generate_code.sh index 7d8ef7970b..5374b12b2b 100755 --- a/k8s/generate_code.sh +++ b/k8s/generate_code.sh @@ -37,14 +37,14 @@ if [[ ! -d "${GOPATH}/src/k8s.io/code-generator" ]]; then fi # https://github.com/kubernetes-sigs/controller-tools/tree/v0.7.0/cmd/controller-gen -if ! command -v controller-gen > /dev/null; then +if ! command -v ${GOPATH}/bin/controller-gen > /dev/null; then echo "controller-gen is missing" echo "Prepare to install controller-gen" go install sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSION} fi # https://github.com/kubernetes-sigs/kustomize/tree/kustomize/v3.10.0/kustomize -if ! command -v kustomize > /dev/null; then +if ! command -v ${GOPATH}/bin/kustomize > /dev/null; then echo "kustomize is missing" echo "Prepare to install kustomize" mkdir -p ${GOPATH}/src/github.com/kubernetes-sigs @@ -65,16 +65,16 @@ bash ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \ $@ echo Generating CRD -controller-gen crd paths=${APIS_DIR}/... output:crd:dir=${CRDS_DIR} +${GOPATH}/bin/controller-gen crd paths=${APIS_DIR}/... output:crd:dir=${CRDS_DIR} pushd ${CRDS_DIR} -kustomize create --autodetect 2>/dev/null || true -kustomize edit add label longhorn-manager: 2>/dev/null || true +${GOPATH}/bin/kustomize create --autodetect 2>/dev/null || true +${GOPATH}/bin/kustomize edit add label longhorn-manager: 2>/dev/null || true if [ -e ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/patches/crd ]; then cp -a ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/patches/crd patches - find patches -type f | xargs -i sh -c 'kustomize edit add patch --path {}' + find patches -type f | xargs -i sh -c '${GOPATH}/bin/kustomize edit add patch --path {}' fi popd echo "# Generated by the CRDs from ${APIS_DIR}" > ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml -kustomize build ${CRDS_DIR} >> ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml +${GOPATH}/bin/kustomize build ${CRDS_DIR} >> ${GOPATH}/src/${LH_MANAGER_DIR}/k8s/crds.yaml rm -r ${CRDS_DIR} diff --git a/k8s/pkg/apis/longhorn/v1beta2/backup.go b/k8s/pkg/apis/longhorn/v1beta2/backup.go index 45dd5dc59a..5ffdfd7a07 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backup.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backup.go @@ -35,6 +35,9 @@ type BackupSpec struct { // The labels of snapshot backup. // +optional Labels map[string]string `json:"labels"` + // The parameters of snapshot backup. + // +optional + Parameters map[string]string `json:"parameters"` } // BackupStatus defines the observed state of the Longhorn backup @@ -97,6 +100,12 @@ type BackupStatus struct { // Compression method // +optional CompressionMethod BackupCompressionMethod `json:"compressionMethod"` + // Size of newly uploaded data + // +optional + NewlyUploadedDataSize string `json:"newlyUploadDataSize"` + // Size of reuploaded data + // +optional + ReUploadedDataSize string `json:"reUploadedDataSize"` } // +genclient diff --git a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go index 47aeeb3a26..99f7492c94 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go +++ b/k8s/pkg/apis/longhorn/v1beta2/backupvolume.go @@ -55,6 +55,9 @@ type BackupVolumeStatus struct { // +optional // +nullable LastSyncedAt metav1.Time `json:"lastSyncedAt"` + // The number of backups that have been created. + // +optional + BackupCount string `json:"backupCount"` } // +genclient diff --git a/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go b/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go index 4fd1f07ea3..35985018b2 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go +++ b/k8s/pkg/apis/longhorn/v1beta2/recurringjob.go @@ -53,6 +53,9 @@ type RecurringJobSpec struct { // The label of the snapshot/backup. // +optional Labels map[string]string `json:"labels,omitempty"` + // The parameters of the snapshot/backup. + // +optional + Parameters map[string]string `json:"parameters,omitempty"` } // RecurringJobStatus defines the observed state of the Longhorn recurring job diff --git a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go index 48cd18dada..5690244b5d 100644 --- a/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go +++ b/k8s/pkg/apis/longhorn/v1beta2/zz_generated.deepcopy.go @@ -632,6 +632,13 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { (*out)[key] = val } } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } @@ -1894,6 +1901,13 @@ func (in *RecurringJobSpec) DeepCopyInto(out *RecurringJobSpec) { (*out)[key] = val } } + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/manager/engine.go b/manager/engine.go index 7f6e023a1a..eac74a8911 100644 --- a/manager/engine.go +++ b/manager/engine.go @@ -244,7 +244,7 @@ func (m *VolumeManager) PurgeSnapshot(volumeName string) error { return nil } -func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName string, labels map[string]string) error { +func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName string, labels map[string]string, parameters map[string]string) error { if volumeName == "" || snapshotName == "" { return fmt.Errorf("volume and snapshot name required") } @@ -260,6 +260,7 @@ func (m *VolumeManager) BackupSnapshot(backupName, volumeName, snapshotName stri Spec: longhorn.BackupSpec{ SnapshotName: snapshotName, Labels: labels, + Parameters: parameters, }, } _, err := m.ds.CreateBackup(backupCR, volumeName) diff --git a/types/types.go b/types/types.go index 527e94f4f3..8bcb2f9a46 100644 --- a/types/types.go +++ b/types/types.go @@ -207,6 +207,10 @@ const ( StorageNetworkInterface = "lhnet1" ) +const ( + RecurringJobBackupParameterFullBackupInterval = "full-backup-interval" +) + const ( KubernetesMinVersion = "v1.18.0" ) diff --git a/util/util.go b/util/util.go index e6612e242c..d48d749bb2 100644 --- a/util/util.go +++ b/util/util.go @@ -47,6 +47,7 @@ import ( lhns "github.com/longhorn/go-common-libs/ns" lhtypes "github.com/longhorn/go-common-libs/types" + lhbackup "github.com/longhorn/go-common-libs/backup" longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ) @@ -502,6 +503,29 @@ func ValidateSnapshotLabels(labels map[string]string) (map[string]string, error) return validLabels, nil } +func ValidateBackupParameters(parameters map[string]string) error { + for key, value := range parameters { + if err := validateBackupParameter(key, value); err != nil { + return err + } + } + return nil +} + +func validateBackupParameter(key, value string) error { + switch key { + case lhbackup.LonghornBackupParameterBackupMode: + if value != lhbackup.LonghornBackupModeFull && + value != lhbackup.LonghornBackupModeIncremental { + return fmt.Errorf("%v:%v is not a valid option", key, value) + } + default: + return fmt.Errorf("%v:%v is not a valid option", key, value) + } + + return nil +} + func ValidateTags(inputTags []string) ([]string, error) { foundTags := make(map[string]struct{}) var tags []string diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md index 8c780afd6e..acd3058b77 100644 --- a/vendor/github.com/RoaringBitmap/roaring/README.md +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -43,7 +43,7 @@ The ``roaring`` Go library is used by * [M3](https://github.com/m3db/m3) * [trident](https://github.com/NetApp/trident) * [Husky](https://www.datadoghq.com/blog/engineering/introducing-husky/) - +* [FrostDB](https://github.com/polarsignals/frostdb) This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go index a575caff83..80fa676ef6 100644 --- a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -664,10 +664,54 @@ func (ac *arrayContainer) iandNot(a container) container { } func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { - rcb := rc.toBitmapContainer() - acb := ac.toBitmapContainer() - acb.iandNotBitmapSurely(rcb) - *ac = *(acb.toArrayContainer()) + // Fast path: if either the array container or the run container is empty, the result is the array. + if ac.isEmpty() || rc.isEmpty() { + // Empty + return ac + } + // Fast path: if the run container is full, the result is empty. + if rc.isFull() { + ac.content = ac.content[:0] + return ac + } + current_run := 0 + // All values in [start_run, end_end] are part of the run + start_run := rc.iv[current_run].start + end_end := start_run + rc.iv[current_run].length + // We are going to read values in the array at index i, and we are + // going to write them at index pos. So we do in-place processing. + // We always have that pos <= i by construction. So we can either + // overwrite a value just read, or a value that was previous read. + pos := 0 + i := 0 + for ; i < len(ac.content); i++ { + if ac.content[i] < start_run { + // the value in the array appears before the run [start_run, end_end] + ac.content[pos] = ac.content[i] + pos++ + } else if ac.content[i] <= end_end { + // nothing to do, the value is in the array but also in the run. + } else { + // We have the value in the array after the run. We cannot tell + // whether we need to keep it or not. So let us move to another run. + if current_run+1 < len(rc.iv) { + current_run++ + start_run = rc.iv[current_run].start + end_end = start_run + rc.iv[current_run].length + i-- // retry with the same i + } else { + // We have exhausted the number of runs. We can keep the rest of the values + // from i to len(ac.content) - 1 inclusively. + break // We are done, the rest of the array will be kept + } + } + } + for ; i < len(ac.content); i++ { + ac.content[pos] = ac.content[i] + pos++ + } + // We 'shink' the slice. + ac.content = ac.content[:pos] return ac } diff --git a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go index f4829a62d0..7098ba28fc 100644 --- a/vendor/github.com/RoaringBitmap/roaring/runcontainer.go +++ b/vendor/github.com/RoaringBitmap/roaring/runcontainer.go @@ -47,6 +47,7 @@ import ( // runContainer16 does run-length encoding of sets of // uint16 integers. type runContainer16 struct { + // iv is a slice of sorted, non-overlapping, non-adjacent intervals. iv []interval16 } diff --git a/vendor/github.com/bits-and-blooms/bitset/README.md b/vendor/github.com/bits-and-blooms/bitset/README.md index 848234e2fc..fe7bca65eb 100644 --- a/vendor/github.com/bits-and-blooms/bitset/README.md +++ b/vendor/github.com/bits-and-blooms/bitset/README.md @@ -127,6 +127,13 @@ E.g., The memory usage of a bitset using `N` bits is at least `N/8` bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). +The `roaring` library allows you to go back and forth between compressed Roaring bitmaps and the conventional bitset instances: +```Go + mybitset := roaringbitmap.ToBitSet() + newroaringbitmap := roaring.FromBitSet(mybitset) +``` + + ## Implementation Note Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. diff --git a/vendor/github.com/bits-and-blooms/bitset/bitset.go b/vendor/github.com/bits-and-blooms/bitset/bitset.go index 8fb9e9fa20..9f38ed3a9d 100644 --- a/vendor/github.com/bits-and-blooms/bitset/bitset.go +++ b/vendor/github.com/bits-and-blooms/bitset/bitset.go @@ -94,17 +94,17 @@ func (b *BitSet) SetBitsetFrom(buf []uint64) { b.set = buf } -// From is a constructor used to create a BitSet from an array of integers +// From is a constructor used to create a BitSet from an array of words func From(buf []uint64) *BitSet { return FromWithLength(uint(len(buf))*64, buf) } -// FromWithLength constructs from an array of integers and length. +// FromWithLength constructs from an array of words and length. func FromWithLength(len uint, set []uint64) *BitSet { return &BitSet{len, set} } -// Bytes returns the bitset as array of integers +// Bytes returns the bitset as array of words func (b *BitSet) Bytes() []uint64 { return b.set } @@ -551,6 +551,18 @@ func (b *BitSet) ClearAll() *BitSet { return b } +// SetAll sets the entire BitSet +func (b *BitSet) SetAll() *BitSet { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = allBits + } + + b.cleanLastWord() + } + return b +} + // wordCount returns the number of words used in a bit set func (b *BitSet) wordCount() int { return wordsNeededUnbound(b.length) @@ -944,7 +956,8 @@ func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { return b.Count() > other.Count() && b.IsSuperSet(other) } -// DumpAsBits dumps a bit set as a string of bits +// DumpAsBits dumps a bit set as a string of bits. Following the usual convention in Go, +// the least significant bits are printed last (index 0 is at the end of the string). func (b *BitSet) DumpAsBits() string { if b.set == nil { return "." @@ -1135,3 +1148,37 @@ func (b *BitSet) UnmarshalJSON(data []byte) error { _, err = b.ReadFrom(bytes.NewReader(buf)) return err } + +// Rank returns the nunber of set bits up to and including the index +// that are set in the bitset. +// See https://en.wikipedia.org/wiki/Ranking#Ranking_in_statistics +func (b *BitSet) Rank(index uint) uint { + if index >= b.length { + return b.Count() + } + leftover := (index + 1) & 63 + answer := uint(popcntSlice(b.set[:(index+1)>>6])) + if leftover != 0 { + answer += uint(popcount(b.set[(index+1)>>6] << (64 - leftover))) + } + return answer +} + +// Select returns the index of the jth set bit, where j is the argument. +// The caller is responsible to ensure that 0 <= j < Count(): when j is +// out of range, the function returns the length of the bitset (b.length). +// +// Note that this function differs in convention from the Rank function which +// returns 1 when ranking the smallest value. We follow the conventional +// textbook definition of Select and Rank. +func (b *BitSet) Select(index uint) uint { + leftover := index + for idx, word := range b.set { + w := uint(popcount(word)) + if w > leftover { + return uint(idx)*64 + select64(word, leftover) + } + leftover -= w + } + return b.length +} diff --git a/vendor/github.com/bits-and-blooms/bitset/select.go b/vendor/github.com/bits-and-blooms/bitset/select.go new file mode 100644 index 0000000000..f15e74a2c9 --- /dev/null +++ b/vendor/github.com/bits-and-blooms/bitset/select.go @@ -0,0 +1,45 @@ +package bitset + +func select64(w uint64, j uint) uint { + seen := 0 + // Divide 64bit + part := w & 0xFFFFFFFF + n := uint(popcount(part)) + if n <= j { + part = w >> 32 + seen += 32 + j -= n + } + ww := part + + // Divide 32bit + part = ww & 0xFFFF + + n = uint(popcount(part)) + if n <= j { + part = ww >> 16 + seen += 16 + j -= n + } + ww = part + + // Divide 16bit + part = ww & 0xFF + n = uint(popcount(part)) + if n <= j { + part = ww >> 8 + seen += 8 + j -= n + } + ww = part + + // Lookup in final byte + counter := 0 + for ; counter < 8; counter++ { + j -= uint((ww >> counter) & 1) + if j+1 == 0 { + break + } + } + return uint(seen + counter) +} diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go index b480056734..42bf32aab0 100644 --- a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/md2man.go @@ -9,6 +9,8 @@ func Render(doc []byte) []byte { renderer := NewRoffRenderer() return blackfriday.Run(doc, - []blackfriday.Option{blackfriday.WithRenderer(renderer), - blackfriday.WithExtensions(renderer.GetExtensions())}...) + []blackfriday.Option{ + blackfriday.WithRenderer(renderer), + blackfriday.WithExtensions(renderer.GetExtensions()), + }...) } diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go index be2b343606..8a290f1972 100644 --- a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go @@ -1,6 +1,8 @@ package md2man import ( + "bufio" + "bytes" "fmt" "io" "os" @@ -20,34 +22,35 @@ type roffRenderer struct { } const ( - titleHeader = ".TH " - topLevelHeader = "\n\n.SH " - secondLevelHdr = "\n.SH " - otherHeader = "\n.SS " - crTag = "\n" - emphTag = "\\fI" - emphCloseTag = "\\fP" - strongTag = "\\fB" - strongCloseTag = "\\fP" - breakTag = "\n.br\n" - paraTag = "\n.PP\n" - hruleTag = "\n.ti 0\n\\l'\\n(.lu'\n" - linkTag = "\n\\[la]" - linkCloseTag = "\\[ra]" - codespanTag = "\\fB\\fC" - codespanCloseTag = "\\fR" - codeTag = "\n.PP\n.RS\n\n.nf\n" - codeCloseTag = "\n.fi\n.RE\n" - quoteTag = "\n.PP\n.RS\n" - quoteCloseTag = "\n.RE\n" - listTag = "\n.RS\n" - listCloseTag = "\n.RE\n" - dtTag = "\n.TP\n" - dd2Tag = "\n" - tableStart = "\n.TS\nallbox;\n" - tableEnd = ".TE\n" - tableCellStart = "T{\n" - tableCellEnd = "\nT}\n" + titleHeader = ".TH " + topLevelHeader = "\n\n.SH " + secondLevelHdr = "\n.SH " + otherHeader = "\n.SS " + crTag = "\n" + emphTag = "\\fI" + emphCloseTag = "\\fP" + strongTag = "\\fB" + strongCloseTag = "\\fP" + breakTag = "\n.br\n" + paraTag = "\n.PP\n" + hruleTag = "\n.ti 0\n\\l'\\n(.lu'\n" + linkTag = "\n\\[la]" + linkCloseTag = "\\[ra]" + codespanTag = "\\fB" + codespanCloseTag = "\\fR" + codeTag = "\n.EX\n" + codeCloseTag = ".EE\n" // Do not prepend a newline character since code blocks, by definition, include a newline already (or at least as how blackfriday gives us on). + quoteTag = "\n.PP\n.RS\n" + quoteCloseTag = "\n.RE\n" + listTag = "\n.RS\n" + listCloseTag = "\n.RE\n" + dtTag = "\n.TP\n" + dd2Tag = "\n" + tableStart = "\n.TS\nallbox;\n" + tableEnd = ".TE\n" + tableCellStart = "T{\n" + tableCellEnd = "\nT}\n" + tablePreprocessor = `'\" t` ) // NewRoffRenderer creates a new blackfriday Renderer for generating roff documents @@ -74,6 +77,16 @@ func (r *roffRenderer) GetExtensions() blackfriday.Extensions { // RenderHeader handles outputting the header at document start func (r *roffRenderer) RenderHeader(w io.Writer, ast *blackfriday.Node) { + // We need to walk the tree to check if there are any tables. + // If there are, we need to enable the roff table preprocessor. + ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus { + if node.Type == blackfriday.Table { + out(w, tablePreprocessor+"\n") + return blackfriday.Terminate + } + return blackfriday.GoToNext + }) + // disable hyphenation out(w, ".nh\n") } @@ -86,8 +99,7 @@ func (r *roffRenderer) RenderFooter(w io.Writer, ast *blackfriday.Node) { // RenderNode is called for each node in a markdown document; based on the node // type the equivalent roff output is sent to the writer func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus { - - var walkAction = blackfriday.GoToNext + walkAction := blackfriday.GoToNext switch node.Type { case blackfriday.Text: @@ -109,9 +121,16 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering out(w, strongCloseTag) } case blackfriday.Link: - if !entering { - out(w, linkTag+string(node.LinkData.Destination)+linkCloseTag) + // Don't render the link text for automatic links, because this + // will only duplicate the URL in the roff output. + // See https://daringfireball.net/projects/markdown/syntax#autolink + if !bytes.Equal(node.LinkData.Destination, node.FirstChild.Literal) { + out(w, string(node.FirstChild.Literal)) } + // Hyphens in a link must be escaped to avoid word-wrap in the rendered man page. + escapedLink := strings.ReplaceAll(string(node.LinkData.Destination), "-", "\\-") + out(w, linkTag+escapedLink+linkCloseTag) + walkAction = blackfriday.SkipChildren case blackfriday.Image: // ignore images walkAction = blackfriday.SkipChildren @@ -160,6 +179,11 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering r.handleTableCell(w, node, entering) case blackfriday.HTMLSpan: // ignore other HTML tags + case blackfriday.HTMLBlock: + if bytes.HasPrefix(node.Literal, []byte("