diff --git a/CHANGELOG.md b/CHANGELOG.md index 62780bdd9..3ba2049dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ **Fixes:** - Modified log messages about ONTAP media types not associated with performance classes (Issue [#158](https://github.com/NetApp/trident/issues/158)). +- **Docker:** Resolved issue where containers might not restart after restarting Docker (Issue [#160](https://github.com/NetApp/trident/issues/160)). **Enhancements:** - Added ability to set snapshotReserve in backend config files, volume creation options, and PVC annotations (Issue [#43](https://github.com/NetApp/trident/issues/43)). diff --git a/frontend/docker/plugin.go b/frontend/docker/plugin.go index 0f577111f..4d86f187e 100644 --- a/frontend/docker/plugin.go +++ b/frontend/docker/plugin.go @@ -11,10 +11,12 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/docker/go-plugins-helpers/volume" log "github.com/sirupsen/logrus" + "github.com/cenkalti/backoff" "github.com/netapp/trident/config" "github.com/netapp/trident/core" frontendcommon "github.com/netapp/trident/frontend/common" @@ -22,6 +24,10 @@ import ( "github.com/netapp/trident/utils" ) +const ( + startupTimeout = 50 * time.Second +) + type Plugin struct { orchestrator core.Orchestrator driverName string @@ -73,12 +79,14 @@ func registerDockerVolumePlugin(root string) error { return nil } -func getDockerVersion() (*Version, error) { +func (p *Plugin) initDockerVersion() { + + time.Sleep(5 * time.Second) // Get Docker version out, err := exec.Command("docker", "version", "--format", "'{{json .}}'").CombinedOutput() if err != nil { - return nil, err + log.Errorf("could not get Docker version: %v", err) } versionJSON := string(out) versionJSON = strings.TrimSpace(versionJSON) @@ -88,7 +96,7 @@ func getDockerVersion() (*Version, error) { var version Version err = json.Unmarshal([]byte(versionJSON), &version) if err != nil { - return nil, err + log.Errorf("could not parse Docker version: %v", err) } log.WithFields(log.Fields{ @@ -102,11 +110,15 @@ func getDockerVersion() (*Version, error) { "clientOS": version.Server.Os, }).Debug("Docker version info.") - return &version, nil + p.version = &version + config.OrchestratorTelemetry.PlatformVersion = version.Server.Version } func (p *Plugin) Activate() error { + handler := volume.NewHandler(p) + + // Start serving requests on a different thread go func() { var err error if p.driverPort != "" { @@ -128,6 +140,10 @@ func (p *Plugin) Activate() error { log.Fatalf("Failed to activate Docker frontend: %v", err) } }() + + // Read the Docker version on a different thread so we don't deadlock if Docker is also initializing + go p.initDockerVersion() + return nil } @@ -142,16 +158,8 @@ func (p *Plugin) GetName() string { func (p *Plugin) Version() string { - // Get the Docker version on demand if p.version == nil { - - version, err := getDockerVersion() - if err != nil { - log.Errorf("Failed to get the Docker version: %v", err) - return "unknown" - } - - p.version = version + return "unknown" } return p.version.Server.Version @@ -199,7 +207,7 @@ func (p *Plugin) List() (*volume.ListResponse, error) { "method": "List", }).Debug("Docker frontend method is invoked.") - err := p.orchestrator.ReloadVolumes() + err := p.reloadVolumes() if err != nil { return &volume.ListResponse{}, p.dockerError(err) } @@ -228,7 +236,7 @@ func (p *Plugin) Get(request *volume.GetRequest) (*volume.GetResponse, error) { // Get is called at the start of every 'docker volume' workflow except List & Unmount, // so refresh the volume list here. - err := p.orchestrator.ReloadVolumes() + err := p.reloadVolumes() if err != nil { return &volume.GetResponse{}, p.dockerError(err) } @@ -404,3 +412,36 @@ func (p *Plugin) dockerError(err error) error { return err } } + +// reloadVolumes instructs Trident core to refresh its cached volume info from its +// backend storage controller(s). If Trident isn't ready, it will retry for nearly +// the Docker timeout of 60 seconds. Otherwise, it returns immediately with any +// other error or nil if the operation succeeded. +func (p *Plugin) reloadVolumes() error { + + reloadVolumesFunc := func() error { + + err := p.orchestrator.ReloadVolumes() + if err == nil { + return nil + } else if core.IsNotReadyError(err) { + return err + } else { + return backoff.Permanent(err) + } + } + reloadNotify := func(err error, duration time.Duration) { + log.WithFields(log.Fields{ + "increment": duration, + "message": err.Error(), + }).Debugf("Docker frontend waiting to reload volumes.") + } + reloadBackoff := backoff.NewExponentialBackOff() + reloadBackoff.InitialInterval = 1 * time.Second + reloadBackoff.RandomizationFactor = 0.0 + reloadBackoff.Multiplier = 1.0 + reloadBackoff.MaxInterval = 1 * time.Second + reloadBackoff.MaxElapsedTime = startupTimeout + + return backoff.RetryNotify(reloadVolumesFunc, reloadBackoff, reloadNotify) +} diff --git a/storage_drivers/eseries/api/eseries.go b/storage_drivers/eseries/api/eseries.go index 8e180becc..b89d8912f 100644 --- a/storage_drivers/eseries/api/eseries.go +++ b/storage_drivers/eseries/api/eseries.go @@ -93,19 +93,20 @@ func NewAPIClient(config ClientConfig) *Client { } c.config.CompiledPoolNameSearchPattern = compiledRegex - volumeTags = []VolumeTag{ - {"IF", c.config.Protocol}, - {"version", c.config.Telemetry["version"]}, - {"platform", c.config.Telemetry["platform"]}, - {"platformVersion", c.config.Telemetry["platformVersion"]}, - {"plugin", c.config.Telemetry["plugin"]}, - {"storagePrefix", c.config.Telemetry["storagePrefix"]}, - } - return c } -var volumeTags []VolumeTag +func (d Client) makeVolumeTags() []VolumeTag { + + return []VolumeTag{ + {"IF", d.config.Protocol}, + {"version", d.config.Telemetry["version"]}, + {"platform", tridentconfig.OrchestratorTelemetry.Platform}, + {"platformVersion", tridentconfig.OrchestratorTelemetry.PlatformVersion}, + {"plugin", d.config.Telemetry["plugin"]}, + {"storagePrefix", d.config.Telemetry["storagePrefix"]}, + } +} // InvokeAPI makes a REST call to the Web Services Proxy. The body must be a marshaled JSON byte array (or nil). // The method is the HTTP verb (i.e. GET, POST, ...). The resource path is appended to the base URL to identify @@ -608,7 +609,7 @@ func (d Client) CreateVolume( } // Copy static volume metadata and add fstype - tags := append([]VolumeTag(nil), volumeTags...) + tags := d.makeVolumeTags() tags = append(tags, VolumeTag{"fstype", fstype}) // Set up the volume create request diff --git a/storage_drivers/eseries/eseries_iscsi.go b/storage_drivers/eseries/eseries_iscsi.go index f3da1e63c..f610e9963 100644 --- a/storage_drivers/eseries/eseries_iscsi.go +++ b/storage_drivers/eseries/eseries_iscsi.go @@ -102,8 +102,6 @@ func (d *SANStorageDriver) Initialize( telemetry := make(map[string]string) telemetry["version"] = tridentconfig.OrchestratorVersion.ShortString() - telemetry["platform"] = tridentconfig.OrchestratorTelemetry.Platform - telemetry["platformVersion"] = tridentconfig.OrchestratorTelemetry.PlatformVersion telemetry["plugin"] = d.Name() telemetry["storagePrefix"] = *d.Config.StoragePrefix diff --git a/storage_drivers/ontap/ontap_common.go b/storage_drivers/ontap/ontap_common.go index c3e693c38..81e6b0e72 100644 --- a/storage_drivers/ontap/ontap_common.go +++ b/storage_drivers/ontap/ontap_common.go @@ -16,7 +16,7 @@ import ( "github.com/cenkalti/backoff" log "github.com/sirupsen/logrus" - trident "github.com/netapp/trident/config" + tridentconfig "github.com/netapp/trident/config" "github.com/netapp/trident/storage" sa "github.com/netapp/trident/storage_attribute" drivers "github.com/netapp/trident/storage_drivers" @@ -32,7 +32,7 @@ const ( ) type Telemetry struct { - trident.Telemetry + tridentconfig.Telemetry Plugin string `json:"plugin"` SVM string `json:"svm"` StoragePrefix string `json:"storagePrefix"` @@ -50,7 +50,7 @@ type StorageDriver interface { // InitializeOntapConfig parses the ONTAP config, mixing in the specified common config. func InitializeOntapConfig( - context trident.DriverContext, configJSON string, commonConfig *drivers.CommonStorageDriverConfig, + context tridentconfig.DriverContext, configJSON string, commonConfig *drivers.CommonStorageDriverConfig, ) (*drivers.OntapStorageDriverConfig, error) { if commonConfig.DebugTraceFlags["method"] { @@ -75,7 +75,6 @@ func InitializeOntapConfig( func NewOntapTelemetry(d StorageDriver) *Telemetry { t := &Telemetry{ - Telemetry: trident.OrchestratorTelemetry, Plugin: d.Name(), SVM: d.GetConfig().SVM, StoragePrefix: *d.GetConfig().StoragePrefix, @@ -488,7 +487,7 @@ func EMSHeartbeat(driver StorageDriver) { emsResponse, err := driver.GetAPI().EmsAutosupportLog( strconv.Itoa(drivers.ConfigVersion), false, "heartbeat", hostname, - string(message), 1, trident.OrchestratorName, 5) + string(message), 1, tridentconfig.OrchestratorName, 5) if err = api.GetError(emsResponse, err); err != nil { log.WithFields(log.Fields{ @@ -1017,7 +1016,7 @@ func getVolumeOptsCommon( func getInternalVolumeNameCommon(commonConfig *drivers.CommonStorageDriverConfig, name string) string { - if trident.UsingPassthroughStore { + if tridentconfig.UsingPassthroughStore { // With a passthrough store, the name mapping must remain reversible return *commonConfig.StoragePrefix + name } else { diff --git a/storage_drivers/ontap/ontap_nas.go b/storage_drivers/ontap/ontap_nas.go index d113d69cd..5eb79c394 100644 --- a/storage_drivers/ontap/ontap_nas.go +++ b/storage_drivers/ontap/ontap_nas.go @@ -36,6 +36,7 @@ func (d *NASStorageDriver) GetAPI() *api.Client { } func (d *NASStorageDriver) GetTelemetry() *Telemetry { + d.Telemetry.Telemetry = tridentconfig.OrchestratorTelemetry return d.Telemetry } diff --git a/storage_drivers/ontap/ontap_nas_flexgroup.go b/storage_drivers/ontap/ontap_nas_flexgroup.go index ebaae5a91..05bf028c3 100644 --- a/storage_drivers/ontap/ontap_nas_flexgroup.go +++ b/storage_drivers/ontap/ontap_nas_flexgroup.go @@ -35,6 +35,7 @@ func (d *NASFlexGroupStorageDriver) GetAPI() *api.Client { } func (d *NASFlexGroupStorageDriver) GetTelemetry() *Telemetry { + d.Telemetry.Telemetry = tridentconfig.OrchestratorTelemetry return d.Telemetry } diff --git a/storage_drivers/ontap/ontap_nas_qtree.go b/storage_drivers/ontap/ontap_nas_qtree.go index bcff9cc36..6288ce06e 100644 --- a/storage_drivers/ontap/ontap_nas_qtree.go +++ b/storage_drivers/ontap/ontap_nas_qtree.go @@ -62,6 +62,7 @@ func (d *NASQtreeStorageDriver) GetAPI() *api.Client { } func (d *NASQtreeStorageDriver) GetTelemetry() *Telemetry { + d.Telemetry.Telemetry = tridentconfig.OrchestratorTelemetry return d.Telemetry } diff --git a/storage_drivers/ontap/ontap_san.go b/storage_drivers/ontap/ontap_san.go index 25ea19514..2db21b9a7 100644 --- a/storage_drivers/ontap/ontap_san.go +++ b/storage_drivers/ontap/ontap_san.go @@ -45,6 +45,7 @@ func (d *SANStorageDriver) GetAPI() *api.Client { } func (d *SANStorageDriver) GetTelemetry() *Telemetry { + d.Telemetry.Telemetry = tridentconfig.OrchestratorTelemetry return d.Telemetry } diff --git a/storage_drivers/solidfire/solidfire_san.go b/storage_drivers/solidfire/solidfire_san.go index c593ce58e..7f90df697 100644 --- a/storage_drivers/solidfire/solidfire_san.go +++ b/storage_drivers/solidfire/solidfire_san.go @@ -39,7 +39,6 @@ type SANStorageDriver struct { AccessGroups []int64 LegacyNamePrefix string InitiatorIFace string - Telemetry *Telemetry } type StorageDriverConfigExternal struct { @@ -86,6 +85,13 @@ func parseType(vTypes []api.VolType, typeName string) (qos api.QoS, err error) { return qos, err } +func (d SANStorageDriver) getTelemetry() *Telemetry { + return &Telemetry{ + Telemetry: tridentconfig.OrchestratorTelemetry, + Plugin: d.Name(), + } +} + // Name is for returning the name of this driver func (d SANStorageDriver) Name() string { return drivers.SolidfireSANStorageDriverName @@ -236,11 +242,6 @@ func (d *SANStorageDriver) Initialize( // log cluster node serial numbers asynchronously since the API can take a long time go d.getNodeSerialNumbers(config.CommonStorageDriverConfig) - d.Telemetry = &Telemetry{ - Telemetry: tridentconfig.OrchestratorTelemetry, - Plugin: d.Name(), - } - d.initialized = true return nil } @@ -516,7 +517,7 @@ func (d *SANStorageDriver) Create(name string, sizeBytes uint64, opts map[string var req api.CreateVolumeRequest var qos api.QoS - telemetry, _ := json.Marshal(d.Telemetry) + telemetry, _ := json.Marshal(d.getTelemetry()) var meta = map[string]string{ "trident": string(telemetry), "docker-name": name, @@ -643,7 +644,7 @@ func (d *SANStorageDriver) CreateClone(name, sourceName, snapshotName string, op } var req api.CloneVolumeRequest - telemetry, _ := json.Marshal(d.Telemetry) + telemetry, _ := json.Marshal(d.getTelemetry()) var meta = map[string]string{ "trident": string(telemetry), "docker-name": name,