From bc87424c8de9fa50e336e0af4af17567c285f9fc Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 19 Sep 2024 16:31:39 +0200 Subject: [PATCH] Refactor auto-upgrade functionality Signed-off-by: Danil-Grigorev --- api/v1alpha1/clusterctl_config_types.go | 11 +- .../templates/rancher-turtles-components.yaml | 13 +- ...tles-capi.cattle.io_clusterctlconfigs.yaml | 13 +- go.mod | 2 +- internal/controllers/clusterctl/config.go | 122 +++++++++++++++++- .../clusterctlconfig_controller.go | 37 ++---- internal/sync/provider_sync.go | 48 ++++--- internal/sync/provider_sync_test.go | 11 +- main.go | 2 +- test/e2e/helpers.go | 2 + .../chart-upgrade/chart_upgrade_test.go | 23 ++++ test/framework/turtles.go | 52 ++++++++ 12 files changed, 254 insertions(+), 82 deletions(-) create mode 100644 test/framework/turtles.go diff --git a/api/v1alpha1/clusterctl_config_types.go b/api/v1alpha1/clusterctl_config_types.go index eca18ff7..957abe52 100644 --- a/api/v1alpha1/clusterctl_config_types.go +++ b/api/v1alpha1/clusterctl_config_types.go @@ -30,10 +30,12 @@ const ( //nolint:lll type ClusterctlConfigSpec struct { // Images is a list of image overrided for specified providers - Images []Image `json:"images"` + // +optional + Images []Image `json:"images,omitempty"` // Provider overrides - Providers ProviderList `json:"providers"` + // +optional + Providers ProviderList `json:"providers,omitempty"` } // Provider allows to define providers with known URLs to pull the components. @@ -48,9 +50,8 @@ type Provider struct { // Type is the type of the provider // +required - // +kubebuilder:validation:Enum=infrastructure;core;controlPlane;bootstrap;addon;runtimeextension;ipam - // +kubebuilder:example=infrastructure - ProviderType Type `json:"type"` + // +kubebuilder:example=InfrastructureProvider + Type string `json:"type"` } // ProviderList is a list of providers. diff --git a/charts/rancher-turtles/templates/rancher-turtles-components.yaml b/charts/rancher-turtles/templates/rancher-turtles-components.yaml index b7b29350..8bd16164 100644 --- a/charts/rancher-turtles/templates/rancher-turtles-components.yaml +++ b/charts/rancher-turtles/templates/rancher-turtles-components.yaml @@ -3170,15 +3170,7 @@ spec: type: string type: description: Type is the type of the provider - enum: - - infrastructure - - core - - controlPlane - - bootstrap - - addon - - runtimeextension - - ipam - example: infrastructure + example: InfrastructureProvider type: string url: description: URL of the provider components. Will be used unless @@ -3190,9 +3182,6 @@ spec: - url type: object type: array - required: - - images - - providers type: object type: object x-kubernetes-validations: diff --git a/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml b/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml index 62f5c373..98e26348 100644 --- a/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml +++ b/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml @@ -74,15 +74,7 @@ spec: type: string type: description: Type is the type of the provider - enum: - - infrastructure - - core - - controlPlane - - bootstrap - - addon - - runtimeextension - - ipam - example: infrastructure + example: InfrastructureProvider type: string url: description: URL of the provider components. Will be used unless @@ -94,9 +86,6 @@ spec: - url type: object type: array - required: - - images - - providers type: object type: object x-kubernetes-validations: diff --git a/go.mod b/go.mod index 553b6e42..cf6b08c2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rancher/turtles go 1.22.0 require ( + github.com/blang/semver/v4 v4.0.0 github.com/go-logr/logr v1.4.2 github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 @@ -25,7 +26,6 @@ require ( require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect diff --git a/internal/controllers/clusterctl/config.go b/internal/controllers/clusterctl/config.go index 79e7b7b2..153c41b1 100644 --- a/internal/controllers/clusterctl/config.go +++ b/internal/controllers/clusterctl/config.go @@ -18,13 +18,22 @@ package clusterctl import ( "cmp" + "context" + "fmt" "os" + "slices" + "strings" _ "embed" + "github.com/blang/semver/v4" corev1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" ) var ( @@ -34,14 +43,125 @@ var ( config *corev1.ConfigMap ) +const ( + latest = "latest" +) + func init() { utilruntime.Must(yaml.UnmarshalStrict(configDefault, &config)) } -// Config returns current set of turtles clusterctl overrides. +// ConfigRepository is a direct clusterctl config repository representation. +type ConfigRepository struct { + Providers turtlesv1.ProviderList `json:"providers"` + Images map[string]ConfigImage `json:"images"` +} + +// ConfigImage is a direct clusterctl representation of image config value. +type ConfigImage struct { + // Repository sets the container registry override to pull images from. + Repository string `json:"repository,omitempty"` + + // Tag allows to specify a tag for the images. + Tag string `json:"tag,omitempty"` +} + +// Config returns current set of embedded turtles clusterctl overrides. func Config() *corev1.ConfigMap { configMap := config.DeepCopy() configMap.Namespace = cmp.Or(os.Getenv("POD_NAMESPACE"), "rancher-turtles-system") return configMap } + +// ClusterConfig collects overrides config from the local in-memory state +// and the user-specified ClusterctlConfig overrides layer. +func ClusterConfig(ctx context.Context, c client.Client) (*ConfigRepository, error) { + log := log.FromContext(ctx) + + configMap := Config() + + config := &turtlesv1.ClusterctlConfig{} + if err := c.Get(ctx, client.ObjectKeyFromObject(configMap), config); client.IgnoreNotFound(err) != nil { + log.Error(err, "Unable to collect ClusterctlConfig resource") + + return nil, err + } + + clusterctlConfig := &ConfigRepository{} + if err := yaml.UnmarshalStrict([]byte(configMap.Data["clusterctl.yaml"]), &clusterctlConfig); err != nil { + log.Error(err, "Unable to deserialize initial clusterctl config") + + return nil, err + } + + if clusterctlConfig.Images == nil { + clusterctlConfig.Images = map[string]ConfigImage{} + } + + clusterctlConfig.Providers = append(clusterctlConfig.Providers, config.Spec.Providers...) + + for _, image := range config.Spec.Images { + clusterctlConfig.Images[image.Name] = ConfigImage{ + Tag: image.Tag, + Repository: image.Repository, + } + } + + return clusterctlConfig, nil +} + +// GetProviderVersion collects version of the collected provider overrides state. +// Returns latest if the version is not found. +func (r *ConfigRepository) GetProviderVersion(ctx context.Context, name, providerType string) string { + for _, provider := range r.Providers { + if provider.Name == name && strings.EqualFold(provider.Type, providerType) { + return collectVersion(ctx, provider.URL) + } + } + + return latest +} + +func collectVersion(ctx context.Context, url string) string { + version := strings.Split(url, "/") + slices.Reverse(version) + + if len(version) < 2 { + log.FromContext(ctx).Info("Provider url is invalid for version resolve, defaulting to latest", "url", url) + + return latest + } + + return version[1] +} + +// VerifyProviderVersion checks version against the expected max version, and returns false +// if the version given is newer then the latest in the clusterctlconfig override. +func (r *ConfigRepository) VerifyProviderVersion(providerVersion, expected string) (bool, error) { + if providerVersion == latest { + return true, nil + } + + version, _ := strings.CutPrefix(providerVersion, "v") + + maxVersion, err := semver.Parse(version) + if err != nil { + return false, fmt.Errorf("unable to parse default provider version %s: %w", providerVersion, err) + } + + if expected == latest { + // Latest should be reduced to the actual version set on the clusterctlprovider resource + return false, nil + } + + version, _ = strings.CutPrefix(expected, "v") + + destinatonVersion, err := semver.Parse(version) + if err != nil { + return false, fmt.Errorf("unable to parse provider version %s: %w", expected, err) + } + + // Disallow versions beyond clusterctl.yaml default + return maxVersion.LTE(destinatonVersion), nil +} diff --git a/internal/controllers/clusterctlconfig_controller.go b/internal/controllers/clusterctlconfig_controller.go index 2f6627a6..36a13302 100644 --- a/internal/controllers/clusterctlconfig_controller.go +++ b/internal/controllers/clusterctlconfig_controller.go @@ -87,44 +87,22 @@ func (r *ClusterctlConfigReconciler) SetupWithManager(ctx context.Context, mgr c return nil } -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs,verbs=get;list;watch;patch -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs/status,verbs=get;list;watch;patch -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs/finalizers,verbs=get;list;watch;patch;update +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs,verbs=get;list;watch;patch +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs/status,verbs=get;list;watch;patch +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs/finalizers,verbs=get;list;watch;patch;update //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;patch // Reconcile reconciles the EtcdMachineSnapshot object. -func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { +func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.Result, error) { log := log.FromContext(ctx) - configMap := clusterctl.Config() - - config := &turtlesv1.ClusterctlConfig{} - if err := r.Client.Get(ctx, req.NamespacedName, config); client.IgnoreNotFound(err) != nil { - log.Error(err, "Unable to collect ClusterctlConfig resource") - - return ctrl.Result{}, err - } - - clusterctlConfig := &Config{} - if err := yaml.UnmarshalStrict([]byte(configMap.Data["clusterctl.yaml"]), &clusterctlConfig); err != nil { - log.Error(err, "Unable to deserialize initial clusterctl config") + clusterctlConfig, err := clusterctl.ClusterConfig(ctx, r.Client) + if err != nil { + log.Error(err, "Unable to serialize updated clusterctl config") return ctrl.Result{}, err } - if clusterctlConfig.Images == nil { - clusterctlConfig.Images = map[string]ConfigImage{} - } - - clusterctlConfig.Providers = append(clusterctlConfig.Providers, config.Spec.Providers...) - - for _, image := range config.Spec.Images { - clusterctlConfig.Images[image.Name] = ConfigImage{ - Tag: image.Tag, - Repository: image.Repository, - } - } - clusterctlYaml, err := yaml.Marshal(clusterctlConfig) if err != nil { log.Error(err, "Unable to serialize updated clusterctl config") @@ -132,6 +110,7 @@ func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, req reconcil return ctrl.Result{}, err } + configMap := clusterctl.Config() configMap.Data["clusterctl.yaml"] = string(clusterctlYaml) if err := r.Client.Patch(ctx, configMap, client.Apply, []client.PatchOption{ diff --git a/internal/sync/provider_sync.go b/internal/sync/provider_sync.go index dad56cd1..af41d27d 100644 --- a/internal/sync/provider_sync.go +++ b/internal/sync/provider_sync.go @@ -17,19 +17,22 @@ limitations under the License. package sync import ( + "cmp" "context" + "fmt" "strings" "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2" "sigs.k8s.io/cluster-api/util/conditions" turtlesv1 "github.com/rancher/turtles/api/v1alpha1" "github.com/rancher/turtles/internal/api" + "github.com/rancher/turtles/internal/controllers/clusterctl" ) // AppliedSpecHashAnnotation is a spec hash annotation set by CAPI Operator, @@ -91,9 +94,13 @@ func (ProviderSync) Template(capiProvider *turtlesv1.CAPIProvider) client.Object // Spec -> down // up <- Status. func (s *ProviderSync) Sync(ctx context.Context) error { + if err := s.updateLatestVersion(ctx); err != nil { + return err + } + s.SyncObjects() - return s.updateLatestVersion(ctx) + return nil } // SyncObjects updates the Source CAPIProvider object and the destination provider object states. @@ -153,30 +160,39 @@ func (s *ProviderSync) rolloutInfrastructure() { func (s *ProviderSync) updateLatestVersion(ctx context.Context) error { // Skip for user specified versions + // TODO: We may potentially need to verify if version specified is built in the override, + // and notify user with condition otherwise if s.Source.Spec.Version != "" { return nil } - now := time.Now().UTC() - lastCheck := conditions.Get(s.Source, turtlesv1.CheckLatestVersionTime) + log := log.FromContext(ctx) - if lastCheck != nil && lastCheck.Status == corev1.ConditionTrue && lastCheck.LastTransitionTime.Add(24*time.Hour).After(now) { - return nil + config, err := clusterctl.ClusterConfig(ctx, s.client) + if err != nil { + return err } - patchBase := client.MergeFrom(s.Destination) + providerVersion := config.GetProviderVersion(ctx, cmp.Or(s.Source.Spec.Name, s.Source.Name), s.Source.Spec.Type.ToKind()) + expected := cmp.Or(s.Source.Spec.Version, "latest") - // Unsetting .spec.version to force latest version rollout - spec := s.Destination.GetSpec() - spec.Version = "" - s.Destination.SetSpec(spec) + if valid, err := config.VerifyProviderVersion(providerVersion, expected); err != nil { + return err + } else if !valid { + lastCheck := conditions.Get(s.Source, turtlesv1.CheckLatestVersionTime) + updatedMessage := fmt.Sprintf("Updated to latest %s version", providerVersion) - conditions.MarkTrue(s.Source, turtlesv1.CheckLatestVersionTime) + if lastCheck == nil || lastCheck.Message != updatedMessage { + log.Info(fmt.Sprintf("Version %s is beyound current latest, setting to %s", expected, providerVersion)) - err := s.client.Patch(ctx, s.Destination, patchBase) - if err != nil { - conditions.MarkUnknown(s.Source, turtlesv1.CheckLatestVersionTime, "Requesting latest version rollout", "") + lastCheck = conditions.TrueCondition(turtlesv1.CheckLatestVersionTime) + lastCheck.Message = updatedMessage + + conditions.Set(s.Source, lastCheck) + } + + s.Source.Spec.Version = providerVersion } - return client.IgnoreNotFound(err) + return nil } diff --git a/internal/sync/provider_sync_test.go b/internal/sync/provider_sync_test.go index 7b7ca647..9cfd03db 100644 --- a/internal/sync/provider_sync_test.go +++ b/internal/sync/provider_sync_test.go @@ -92,6 +92,7 @@ var _ = Describe("Provider sync", func() { Conditions: clusterv1.Conditions{ { Type: turtlesv1.CheckLatestVersionTime, + Message: "Updated to latest v1.4.6 version", Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(-23 * time.Hour)), }, @@ -115,6 +116,9 @@ var _ = Describe("Provider sync", func() { It("Should sync spec down", func() { s := sync.NewProviderSync(testEnv, capiProvider.DeepCopy()) + expected := capiProvider.DeepCopy() + expected.Spec.Version = "v1.7.3" + Eventually(func(g Gomega) { g.Expect(s.Get(ctx)).To(Succeed()) g.Expect(s.Sync(ctx)).To(Succeed()) @@ -124,7 +128,7 @@ var _ = Describe("Provider sync", func() { }).Should(Succeed()) Eventually(Object(infrastructure)).Should( - HaveField("Spec.ProviderSpec", Equal(capiProvider.Spec.ProviderSpec))) + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) }) It("Should sync azure spec", func() { @@ -198,13 +202,10 @@ var _ = Describe("Provider sync", func() { s.Apply(ctx, &err) g.Expect(err).ToNot(HaveOccurred()) g.Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(infrastructure), dest)).To(Succeed()) - g.Expect(dest.GetAnnotations()).To(HaveKeyWithValue(sync.AppliedSpecHashAnnotation, "")) - g.Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(infrastructure), dest)).To(Succeed()) g.Expect(capiProvider.Status.Conditions).To(HaveLen(2)) g.Expect(conditions.IsTrue(capiProvider, turtlesv1.LastAppliedConfigurationTime)).To(BeTrue()) g.Expect(conditions.IsTrue(capiProvider, turtlesv1.CheckLatestVersionTime)).To(BeTrue()) - g.Expect(conditions.Get(capiProvider, turtlesv1.CheckLatestVersionTime).LastTransitionTime.Equal( - &lastVersionCheckCondition.LastTransitionTime)).To(BeTrue()) + g.Expect(conditions.Get(capiProvider, turtlesv1.CheckLatestVersionTime).Message).To(Equal("Updated to latest v1.7.3 version")) g.Expect(conditions.Get(capiProvider, turtlesv1.LastAppliedConfigurationTime).LastTransitionTime.After( appliedCondition.LastTransitionTime.Time)).To(BeTrue()) }).Should(Succeed()) diff --git a/main.go b/main.go index b5af5806..b73ce835 100644 --- a/main.go +++ b/main.go @@ -285,7 +285,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Info("enabling Clusterctl Config synchronization controller") if err := (&controllers.ClusterctlConfigReconciler{ - Client: uncachedClient, + Client: mgr.GetClient(), }).SetupWithManager(ctx, mgr, controller.Options{ MaxConcurrentReconciles: concurrencyNumber, CacheSyncTimeout: maxDuration, diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 3773a1d0..e169bdec 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -29,6 +29,7 @@ import ( . "github.com/onsi/gomega" + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -84,6 +85,7 @@ func DumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr func InitScheme() *runtime.Scheme { scheme := runtime.NewScheme() framework.TryAddDefaultSchemes(scheme) + Expect(turtlesv1.AddToScheme(scheme)).To(Succeed()) Expect(operatorv1.AddToScheme(scheme)).To(Succeed()) Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) Expect(provisioningv1.AddToScheme(scheme)).To(Succeed()) diff --git a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go index 17a7b1fb..645d41bd 100644 --- a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go +++ b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go @@ -101,6 +101,29 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) }) + upgradeInput.PostUpgradeSteps = append(upgradeInput.PostUpgradeSteps, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "cluster-api", + Namespace: "capi-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "kubeadm-control-plane", + Namespace: "capi-kubeadm-control-plane-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "docker", + Namespace: "capd-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }) + testenv.UpgradeRancherTurtles(ctx, upgradeInput) }) }) diff --git a/test/framework/turtles.go b/test/framework/turtles.go new file mode 100644 index 00000000..3ac6e4c9 --- /dev/null +++ b/test/framework/turtles.go @@ -0,0 +1,52 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" + capiframework "sigs.k8s.io/cluster-api/test/framework" + + . "github.com/onsi/gomega" +) + +type WaitForCAPIProviderRolloutInput struct { + capiframework.Getter + Name, Namespace, Version string +} + +func WaitForCAPIProviderRollout(ctx context.Context, input WaitForCAPIProviderRolloutInput, intervals ...interface{}) { + capiProvider := &turtlesv1.CAPIProvider{} + key := types.NamespacedName{ + Name: input.Name, + Namespace: input.Namespace, + } + + Byf("Waiting for CAPIProvider %s to be at version %s", key.String(), input.Version) + + Eventually(func(g Gomega) { + g.Expect(input.Getter.Get(ctx, key, capiProvider)).To(Succeed()) + g.Expect(capiProvider.Status.InstalledVersion).ToNot(BeNil()) + g.Expect(*capiProvider.Status.InstalledVersion).To(Equal(input.Version)) + }, intervals...).Should(Succeed(), + "Failed to get CAPIProvider %s with version %s. Last observed: %s", + key.String(), input.Version, klog.KObj(capiProvider)) +}