fix(upgrades): set correct cluster_template_id

Jenkinsfile

@@ -14,6 +14,8 @@ jobs['unit'] = {
 
         sh 'poetry install'
 
+        sh './hack/setup-helm.sh'
+
         try {
             sh 'poetry run pytest --junitxml=junit.xml magnum_cluster_api/tests/unit'
         } finally {

magnum_cluster_api/driver.py

@@ -282,21 +282,44 @@ def upgrade_cluster(
         """
         Upgrade a cluster to a new version of Kubernetes.
         """
-        # TODO: nodegroup?
-
-        cluster_resource = objects.Cluster.for_magnum_cluster(self.k8s_api, cluster)
-
-        resources.apply_cluster_from_magnum_cluster(
-            context, self.k8s_api, cluster, cluster_template=cluster_template
-        )
-
-        # Wait till the generation has been increased
-        cluster_resource.wait_for_observed_generation_changed()
-
-        # NOTE(mnaser): We need to save the cluster status here to make sure
-        #               it happens inside the lock.
-        cluster.status = fields.ClusterStatus.UPDATE_IN_PROGRESS
-        cluster.save()
+        # NOTE(mnaser): The only label that we change during the upgrade is the `kube_tag`
+        #               label.
+        #
+        #               Historically, the upgrade cluster has been a "hammer" that was
+        #               used to sync the Kubernetes Cluster API objects with the Magnum
+        #               objects.  However, by doing this, we're losing the ability to
+        #               maintain the existing labels of the cluster.
+        #
+        #               For now, upgrade cluster simply modifies the labels that are
+        #               necessary for the upgrade, nothing else.  For the future, we
+        #               can perhaps use the `update_cluster` API.
+        current_kube_tag = cluster.labels["kube_tag"]
+        new_kube_tag = cluster_template.labels["kube_tag"]
+
+        # XXX(mnaser): The Magnum API historically only did upgrade one node group at a
+        #              time.  This is a limitation of the Magnum API and not the Magnum
+        #              Cluster API since doing multiple rolling upgrades was not very
+        #              well supported in the past.
+        #
+        #              The Magnum Cluster API does not have this limitation in this case
+        #              we ignore the `nodegroup` parameter and upgrade the entire cluster
+        #              at once.
+        cluster.cluster_template_id = cluster_template.uuid
+        cluster.labels["kube_tag"] = new_kube_tag
+
+        for ng in cluster.nodegroups:
+            ng.status = fields.ClusterStatus.UPDATE_IN_PROGRESS
+            ng.image_id = cluster_template.image_id
+            ng.labels["kube_tag"] = new_kube_tag
+            ng.save()
+
+        if current_kube_tag != new_kube_tag:
+            cluster_resource = objects.Cluster.for_magnum_cluster(self.k8s_api, cluster)
+            resources.apply_cluster_from_magnum_cluster(context, self.k8s_api, cluster)
+            cluster_resource.wait_for_observed_generation_changed()
+
+        # NOTE(mnaser): We do not save the cluster object here because the Magnum driver
+        #               will save the object that it passed to us here.
 
     @cluster_lock_wrapper
     def delete_cluster(self, context, cluster: magnum_objects.Cluster):

magnum_cluster_api/integrations/common.py

@@ -66,7 +66,7 @@ def get_cloud_provider_image(
 
 
 def get_cloud_provider_tag(cluster: objects.Cluster, label: str) -> str:
-    tag_label = utils.get_cluster_label(cluster, label, None)
+    tag_label = cluster.labels.get(label, None)
     if tag_label:
         return tag_label
 

magnum_cluster_api/objects.py

@@ -36,14 +36,18 @@ def events(self):
             },
         )
 
+    @property
+    def observed_generation(self):
+        return self.obj.get("status", {}).get("observedGeneration")
+
     def wait_for_observed_generation_changed(
         self,
         existing_observed_generation: int = 0,
         timeout: int = 10,
         interval: int = 1,
     ):
         if existing_observed_generation == 0:
-            existing_observed_generation = self.obj["status"]["observedGeneration"]
+            existing_observed_generation = self.observed_generation
 
         for attempt in Retrying(
             retry=(
@@ -56,9 +60,7 @@ def wait_for_observed_generation_changed(
             with attempt:
                 self.reload()
             if not attempt.retry_state.outcome.failed:
-                attempt.retry_state.set_result(
-                    self.obj.get("status", {}).get("observedGeneration")
-                )
+                attempt.retry_state.set_result(self.observed_generation)
 
 
 class EndpointSlice(NamespacedAPIObject):
@@ -278,7 +280,6 @@ def openstack_cluster(self):
 
 
 class StorageClass(pykube.objects.APIObject):
-
     version = "storage.k8s.io/v1"
     endpoint = "storageclasses"
     kind = "StorageClass"