From 910a379a499294f32a400fbf345841d2b1a16c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Va=C5=A1ko?= Date: Fri, 19 Mar 2021 01:58:24 +0100 Subject: [PATCH] Implement cinder volume driver as block storage (#9) Implement cinder volume driver as block storage Unrelated changes: - Use fmt.Sprintf to concatenate strings - Rename plugin to common name velero.io/openstack - Allow authenticating against swift with special env vars - Allow Authenticate to specify which service is authenticating - Log enhancements (lowercase the errors) - Update README with installation, env vars usage, volume backups section, development section --- README.md | 111 +++++++++++++++++-- main.go | 10 +- src/cinder/block_store.go | 225 ++++++++++++++++++++++++++++++++++++++ src/swift/object_store.go | 24 ++-- src/utils/auth.go | 31 +++++- 5 files changed, 373 insertions(+), 28 deletions(-) create mode 100644 src/cinder/block_store.go diff --git a/README.md b/README.md index b5dfa7b..14f2cfa 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,57 @@ -# Swift plugin for Velero +# Velero Plugin for Openstack -Openstack Swift plugin for [velero](https://github.com/vmware-tanzu/velero/) backups. +Openstack Cinder and Swift plugin for [velero](https://github.com/vmware-tanzu/velero/) backups. -## Configure +## Configuration -Configure velero container with swift authentication environment variables: +Configure velero container with your Openstack authentication environment variables: ```bash +# Keystone v2.0 export OS_AUTH_URL= export OS_USERNAME= export OS_PASSWORD= export OS_REGION_NAME= +# Keystone v3 +export OS_AUTH_URL= +export OS_PASSWORD= +export OS_USERNAME= +export OS_PROJECT_ID= +export OS_PROJECT_NAME= +export OS_REGION_NAME= +export OS_DOMAIN_NAME= + # If you want to test with unsecure certificates export OS_VERIFY="false" ``` +If your Openstack cloud has separated Swift service (SwiftStack or different), you can specify special environment variables for Swift to authenticate it and keep the standard ones for Cinder: + +```bash +# Swift with SwiftStack +export OS_SWIFT_AUTH_URL= +export OS_SWIFT_PASSWORD= +export OS_SWIFT_PROJECT_ID= +export OS_SWIFT_REGION_NAME= +export OS_SWIFT_TENANT_NAME= +export OS_SWIFT_USERNAME= +``` + +### Install Using Velero CLI + Initialize velero plugin ```bash # Initialize velero from scratch: -velero install --provider swift --plugins lirt/velero-plugin-swift:v0.1.1 --bucket --no-secret +velero install \ + --provider "velero.io/openstack" \ + --plugins lirt/velero-plugin-for-openstack:v0.2.0 \ + --bucket \ + --no-secret # Or add plugin to existing velero: -velero plugin add lirt/velero-plugin-swift:v0.1.1 +velero plugin add lirt/velero-plugin-for-openstack:v0.2.0 ``` Change configuration of `backupstoragelocations.velero.io`: @@ -32,15 +60,68 @@ Change configuration of `backupstoragelocations.velero.io`: spec: objectStorage: bucket: - provider: swift + provider: velero.io/openstack ``` -## Test +Change configuration of `volumesnapshotlocations.velero.io`: + +```yaml + spec: + provider: velero.io/openstack +``` + +### Install Using Helm Chart + +Alternative installation can be done using Helm Charts. + +There is an [official helm chart for Velero](https://github.com/vmware-tanzu/helm-charts/) which can be used to install both velero and velero openstack plugin. + +To use it, first create `values.yaml` file which will later be used in helm installation (here is just minimal necessary configuration): + +```yaml +--- +credentials: + extraSecretRef: "velero-credentials" +configuration: + provider: openstack + backupStorageLocation: + bucket: my-swift-bucket +initContainers: +- name: velero-plugin-openstack + image: lirt/velero-plugin-for-openstack:v0.2.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /target + name: plugins +snapshotsEnabled: true +backupsEnabled: true +# caCert: +``` + +Make sure that secret `velero-credentials` exists and has proper format and content. + +Then install `velero` using command like this: ```bash -go test -v ./... +helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts +helm repo update +helm upgrade \ + velero \ + vmware-tanzu/velero \ + --install \ + --namespace velero \ + --values values.yaml \ + --version 2.15.0 ``` +## Volume Backups + +Please note two things regarding volume backups: +1. The snapshots are done using flag `--force`. The reason is that volumes in state `in-use` cannot be snapshotted without it (they would need to be detached in advance). In some cases this can make snapshot contents inconsistent. +2. Snapshots in the cinder backend are not always supposed to be used as durable. In some cases for proper availability, the snapshot need to be backed up to off-site storage. Please consult if your cinder backend creates durable snapshots with your cloud provider. + +Volume backups with Velero can also be done using [Restic](https://velero.io/docs/main/restic/). + ## Build ```bash @@ -49,5 +130,15 @@ go mod tidy go build # Build image -docker build --file docker/Dockerfile --tag velero-swift:my-test-tag . +docker build --file docker/Dockerfile --tag velero-plugin-for-openstack:my-test-tag . ``` + +## Test + +```bash +go test -v ./... +``` + +## Development + +The plugin interface is built based on the [official Velero plugin example](https://github.com/vmware-tanzu/velero-plugin-example). \ No newline at end of file diff --git a/main.go b/main.go index 4e814c8..e4f7ba0 100644 --- a/main.go +++ b/main.go @@ -1,17 +1,23 @@ package main import ( - swift "github.com/Lirt/velero-plugin-swift/src/swift" + "github.com/Lirt/velero-plugin-swift/src/cinder" + "github.com/Lirt/velero-plugin-swift/src/swift" "github.com/sirupsen/logrus" veleroplugin "github.com/vmware-tanzu/velero/pkg/plugin/framework" ) func main() { veleroplugin.NewServer(). - RegisterObjectStore("velero.io/swift", newSwiftObjectStore). + RegisterObjectStore("velero.io/openstack", newSwiftObjectStore). + RegisterVolumeSnapshotter("velero.io/openstack", newCinderBlockStore). Serve() } func newSwiftObjectStore(logger logrus.FieldLogger) (interface{}, error) { return swift.NewObjectStore(logger), nil } + +func newCinderBlockStore(logger logrus.FieldLogger) (interface{}, error) { + return cinder.NewBlockStore(logger), nil +} diff --git a/src/cinder/block_store.go b/src/cinder/block_store.go new file mode 100644 index 0000000..af5b4eb --- /dev/null +++ b/src/cinder/block_store.go @@ -0,0 +1,225 @@ +package cinder + +import ( + "fmt" + "math/rand" + "strconv" + + "github.com/Lirt/velero-plugin-swift/src/utils" + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +// BlockStore is a plugin for containing state for the Cinder Block Storage +type BlockStore struct { + client *gophercloud.ServiceClient + provider *gophercloud.ProviderClient + config map[string]string + log logrus.FieldLogger +} + +// NewBlockStore instantiates a Cinder Volume Snapshotter. +func NewBlockStore(log logrus.FieldLogger) *BlockStore { + return &BlockStore{log: log} +} + +var _ velero.VolumeSnapshotter = (*BlockStore)(nil) + +// Init prepares the Cinder VolumeSnapshotter for usage using the provided map of +// configuration key-value pairs. It returns an error if the VolumeSnapshotter +// cannot be initialized from the provided config. +func (b *BlockStore) Init(config map[string]string) error { + b.log.Infof("BlockStore.Init called", config) + b.config = config + + // Authenticate to Openstack + err := utils.Authenticate(&b.provider, "cinder", b.log) + if err != nil { + return fmt.Errorf("failed to authenticate against openstack: %v", err) + } + + if b.client == nil { + region := utils.GetEnv("OS_REGION_NAME", "") + b.client, err = openstack.NewBlockStorageV3(b.provider, gophercloud.EndpointOpts{ + Region: region, + }) + if err != nil { + return fmt.Errorf("failed to create cinder storage client: %v", err) + } + } + + return nil +} + +// CreateVolumeFromSnapshot creates a new volume in the specified +// availability zone, initialized from the provided snapshot and with the specified type. +// IOPS is ignored as it is not used in Cinder. +func (b *BlockStore) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { + b.log.Infof("CreateVolumeFromSnapshot called", snapshotID, volumeType, volumeAZ) + snapshotReadyTimeout := 300 + volumeName := fmt.Sprintf("%s.backup.%s", snapshotID, strconv.FormatUint(rand.Uint64(), 10)) + + // Make sure snapshot is in ready state + // Possible values for snapshot state: + // https://github.com/openstack/cinder/blob/master/api-ref/source/v3/volumes-v3-snapshots.inc#volume-snapshots-snapshots + b.log.Infof("Waiting for snapshot to be in 'available' state", snapshotID, snapshotReadyTimeout) + + err := snapshots.WaitForStatus(b.client, snapshotID, "available", snapshotReadyTimeout) + if err != nil { + b.log.Errorf("snapshot didn't get into 'available' state within the time limit", snapshotID, snapshotReadyTimeout) + return "", err + } + b.log.Infof("Snapshot is in 'available' state", snapshotID) + + // Create Cinder Volume from snapshot (backup) + b.log.Infof("Starting to create volume from snapshot") + opts := volumes.CreateOpts{ + Description: "Velero backup from snapshot", + Name: volumeName, + VolumeType: volumeType, + AvailabilityZone: volumeAZ, + SnapshotID: snapshotID, + } + + var cinderVolume *volumes.Volume + cinderVolume, err = volumes.Create(b.client, opts).Extract() + if err != nil { + b.log.Errorf("failed to create volume from snapshot", snapshotID) + return "", errors.WithStack(err) + } + b.log.Infof("Backup volume was created", volumeName, cinderVolume.ID) + + return cinderVolume.ID, nil +} + +// GetVolumeInfo returns type of the specified volume in the given availability zone. +// IOPS is not used as it is not supported by Cinder. +func (b *BlockStore) GetVolumeInfo(volumeID, volumeAZ string) (string, *int64, error) { + b.log.Infof("GetVolumeInfo called", volumeID, volumeAZ) + + volume, err := volumes.Get(b.client, volumeID).Extract() + if err != nil { + b.log.Errorf("failed to get volume %v from Cinder", volumeID) + return "", nil, fmt.Errorf("volume %v not found", volumeID) + } + + return volume.VolumeType, nil, nil +} + +// IsVolumeReady Check if the volume is in one of the ready states. +func (b *BlockStore) IsVolumeReady(volumeID, volumeAZ string) (ready bool, err error) { + b.log.Infof("IsVolumeReady called", volumeID, volumeAZ) + + // Get volume object from Cinder + cinderVolume, err := volumes.Get(b.client, volumeID).Extract() + if err != nil { + b.log.Errorf("failed to get volume %v from Cinder", volumeID) + return false, err + } + + // Ready states: + // https://github.com/openstack/cinder/blob/master/api-ref/source/v3/volumes-v3-volumes.inc#volumes-volumes + if cinderVolume.Status == "available" || cinderVolume.Status == "in-use" { + return true, nil + } + + // Volume is not in one of the "ready" states + return false, fmt.Errorf("volume %v is not in ready state, the status is %v", volumeID, cinderVolume.Status) +} + +// CreateSnapshot creates a snapshot of the specified volume, and applies any provided +// set of tags to the snapshot. +func (b *BlockStore) CreateSnapshot(volumeID, volumeAZ string, tags map[string]string) (string, error) { + b.log.Infof("CreateSnapshot called", volumeID, volumeAZ, tags) + snapshotName := fmt.Sprintf("%s.snap.%s", volumeID, strconv.FormatUint(rand.Uint64(), 10)) + + b.log.Infof("Trying to create snapshot", snapshotName) + opts := snapshots.CreateOpts{ + Name: snapshotName, + Description: "Velero snapshot", + Metadata: tags, + VolumeID: volumeID, + Force: true, + } + + // Note: we will wait for snapshot to be in ready state in CreateVolumeForSnapshot() + createResult, err := snapshots.Create(b.client, opts).Extract() + if err != nil { + return "", errors.WithStack(err) + } + snapshotID := createResult.ID + + b.log.Infof("Snapshot finished successfuly", snapshotName, snapshotID) + return snapshotID, nil +} + +// DeleteSnapshot deletes the specified volume snapshot. +func (b *BlockStore) DeleteSnapshot(snapshotID string) error { + b.log.Infof("DeleteSnapshot called", snapshotID) + + // Delete snapshot from Cinder + b.log.Infof("Deleting Snapshot with ID", snapshotID) + err := snapshots.Delete(b.client, snapshotID).ExtractErr() + if err != nil { + return errors.WithStack(err) + } + + return nil +} + +// GetVolumeID returns the specific identifier for the PersistentVolume. +func (b *BlockStore) GetVolumeID(unstructuredPV runtime.Unstructured) (string, error) { + b.log.Infof("GetVolumeID called", unstructuredPV) + + pv := new(v1.PersistentVolume) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.UnstructuredContent(), pv); err != nil { + return "", errors.WithStack(err) + } + + var volumeID string + + if pv.Spec.Cinder != nil { + volumeID = pv.Spec.Cinder.VolumeID + } else if pv.Spec.CSI.Driver == "cinder.csi.openstack.org" { + volumeID = pv.Spec.CSI.VolumeHandle + } + + if volumeID == "" { + return "", errors.New("volumeID not found") + } + + return volumeID, nil +} + +// SetVolumeID sets the specific identifier for the PersistentVolume. +func (b *BlockStore) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID string) (runtime.Unstructured, error) { + b.log.Infof("SetVolumeID called", unstructuredPV, volumeID) + + pv := new(v1.PersistentVolume) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredPV.UnstructuredContent(), pv); err != nil { + return nil, errors.WithStack(err) + } + + if pv.Spec.Cinder != nil { + pv.Spec.Cinder.VolumeID = volumeID + } else if pv.Spec.CSI.Driver == "cinder.csi.openstack.org" { + pv.Spec.CSI.VolumeHandle = volumeID + } else { + return nil, errors.New("spec.cinder or spec.csi for cinder driver not found") + } + + res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(pv) + if err != nil { + return nil, errors.WithStack(err) + } + + return &unstructured.Unstructured{Object: res}, nil +} diff --git a/src/swift/object_store.go b/src/swift/object_store.go index f4ff706..5a596bd 100644 --- a/src/swift/object_store.go +++ b/src/swift/object_store.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/Lirt/velero-plugin-swift/src/utils" @@ -29,13 +30,16 @@ func NewObjectStore(log logrus.FieldLogger) *ObjectStore { func (o *ObjectStore) Init(config map[string]string) error { o.log.Infof("ObjectStore.Init called") - err := utils.Authenticate(&o.provider, o.log) + err := utils.Authenticate(&o.provider, "swift", o.log) if err != nil { return fmt.Errorf("failed to authenticate against Openstack: %v", err) } if o.client == nil { - region := utils.GetEnv("OS_REGION_NAME", "") + region, ok := os.LookupEnv("OS_SWIFT_REGION_NAME") + if !ok { + region = utils.GetEnv("OS_REGION_NAME", "") + } o.client, err = openstack.NewObjectStorageV1(o.provider, gophercloud.EndpointOpts{ Region: region, }) @@ -57,7 +61,7 @@ func (o *ObjectStore) GetObject(bucket, key string) (io.ReadCloser, error) { object := objects.Download(o.client, bucket, key, nil) if object.Err != nil { - return nil, fmt.Errorf("Failed to download contents of key %v from bucket %v: %v", key, bucket, object.Err) + return nil, fmt.Errorf("failed to download contents of key %v from bucket %v: %v", key, bucket, object.Err) } return object.Body, nil @@ -76,7 +80,7 @@ func (o *ObjectStore) PutObject(bucket string, key string, body io.Reader) error } if _, err := objects.Create(o.client, bucket, key, createOpts).Extract(); err != nil { - return fmt.Errorf("Failed to create new object in bucket %v with key %v: %v", bucket, key, err) + return fmt.Errorf("failed to create new object in bucket %v with key %v: %v", bucket, key, err) } return nil @@ -97,7 +101,7 @@ func (o *ObjectStore) ObjectExists(bucket, key string) (bool, error) { log.Infof("Key %v in bucket %v doesn't exist yet.", key, bucket) return false, nil } - return false, fmt.Errorf("Cannot Get key %v in bucket %v: %v", key, bucket, result.Err) + return false, fmt.Errorf("cannot Get key %v in bucket %v: %v", key, bucket, result.Err) } return true, nil @@ -120,12 +124,12 @@ func (o *ObjectStore) ListCommonPrefixes(bucket, prefix, delimiter string) ([]st allPages, err := objects.List(o.client, bucket, opts).AllPages() if err != nil { - return nil, fmt.Errorf("Failed to list objects in bucket %v: %v", bucket, err) + return nil, fmt.Errorf("failed to list objects in bucket %v: %v", bucket, err) } allObjects, err := objects.ExtractInfo(allPages) if err != nil { - return nil, fmt.Errorf("Failed to extract info from objects in bucket %v: %v", bucket, err) + return nil, fmt.Errorf("failed to extract info from objects in bucket %v: %v", bucket, err) } var objNames []string @@ -146,7 +150,7 @@ func (o *ObjectStore) ListObjects(bucket, prefix string) ([]string, error) { objects, err := o.ListCommonPrefixes(bucket, prefix, "/") if err != nil { - return nil, fmt.Errorf("Failed to list objects from bucket %v with prefix %v: %v", bucket, prefix, err) + return nil, fmt.Errorf("failed to list objects in bucket %v with prefix %v: %v", bucket, prefix, err) } return objects, nil @@ -162,7 +166,7 @@ func (o *ObjectStore) DeleteObject(bucket, key string) error { _, err := objects.Delete(o.client, bucket, key, nil).Extract() if err != nil { - return fmt.Errorf("Failed to delete object with key %v in bucket %v: %v", key, bucket, err) + return fmt.Errorf("failed to delete object with key %v from bucket %v: %v", key, bucket, err) } return nil @@ -181,7 +185,7 @@ func (o *ObjectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (st TTL: int(ttl.Seconds()), }) if err != nil { - return "", fmt.Errorf("Failed to create temporary URL for bucket %v with key %v: %v", bucket, key, err) + return "", fmt.Errorf("failed to create temporary URL for bucket %v with key %v: %v", bucket, key, err) } return url, nil diff --git a/src/utils/auth.go b/src/utils/auth.go index 731be01..4badf7c 100644 --- a/src/utils/auth.go +++ b/src/utils/auth.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "net/http" + "os" "strconv" "github.com/gophercloud/gophercloud" @@ -12,7 +13,7 @@ import ( ) // Authenticate to Openstack and write client result to **pc -func Authenticate(pc **gophercloud.ProviderClient, log logrus.FieldLogger) error { +func Authenticate(pc **gophercloud.ProviderClient, service string, log logrus.FieldLogger) error { // If service client is already initialized and contains auth result // we know we were already authenticated, or the client was reauthenticated // using AllowReauth @@ -23,10 +24,28 @@ func Authenticate(pc **gophercloud.ProviderClient, log logrus.FieldLogger) error } } - log.Infof("Authenticating against Openstack using environment variables") - authOpts, err := openstack.AuthOptionsFromEnv() - if err != nil { - return err + var err error + var authOpts gophercloud.AuthOptions + + if _, ok := os.LookupEnv("OS_SWIFT_AUTH_URL"); ok && service == "swift" { + log.Infof("Authenticating against Swift using environment variables") + authOpts = gophercloud.AuthOptions{ + IdentityEndpoint: os.Getenv("OS_SWIFT_AUTH_URL"), + Username: os.Getenv("OS_SWIFT_USERNAME"), + UserID: os.Getenv("OS_SWIFT_USER_ID"), + Password: os.Getenv("OS_SWIFT_PASSWORD"), + Passcode: os.Getenv("OS_SWIFT_PASSCODE"), + DomainID: os.Getenv("OS_SWIFT_DOMAIN_ID"), + DomainName: os.Getenv("OS_SWIFT_DOMAIN_NAME"), + TenantID: os.Getenv("OS_SWIFT_TENANT_ID"), + TenantName: os.Getenv("OS_SWIFT_TENANT_NAME"), + } + } else { + log.Infof("Authenticating against Openstack using environment variables") + authOpts, err = openstack.AuthOptionsFromEnv() + if err != nil { + return err + } } authOpts.AllowReauth = true @@ -38,7 +57,7 @@ func Authenticate(pc **gophercloud.ProviderClient, log logrus.FieldLogger) error tlsVerify, err := strconv.ParseBool(GetEnv("OS_VERIFY", "true")) if err != nil { - return fmt.Errorf("Cannot parse boolean from OS_VERIFY environment variable: %v", err) + return fmt.Errorf("cannot parse boolean from OS_VERIFY environment variable: %v", err) } tlsconfig := &tls.Config{}