External manifest for image_loader

magnum_cluster_api/cmd/image_loader.py

@@ -17,6 +17,7 @@
 import shutil
 import subprocess
 import tempfile
+import yaml
 
 import click
 import platformdirs
@@ -57,6 +58,10 @@
     required=True,
     help="Target image repository",
 )
+@click.option(
+    "--images_manifest",
+    help="YAML file containing image manifest",
+)
 @click.option(
     "--parallel",
     default=8,
@@ -67,7 +72,7 @@
     is_flag=True,
     help="Allow insecure connections to the registry.",
 )
-def main(repository, parallel, insecure):
+def main(repository, images_manifest, parallel, insecure):
     """
     Load images into a remote registry for `container_infra_prefix` usage.
     """
@@ -79,23 +84,31 @@ def main(repository, parallel, insecure):
              https://github.com/google/go-containerregistry/blob/main/cmd/crane/README.md#installation"""
         )
 
-    # NOTE(mnaser): This list must be maintained manually because the image
-    #               registry must be able to support a few different versions
-    #               of Kubernetes since it is possible to have multiple
-    #               clusters running different versions of Kubernetes at the
-    #               same time.
-    images = set(
-        _get_all_kubeadm_images()
-        + _get_calico_images()
-        + _get_cilium_images()
-        + _get_cloud_provider_images()
-        + _get_infra_images()
-    )
+    if images_manifest:
+        with open(images_manifest, 'r') as file:
+            manifest_data = yaml.safe_load(file)
+
+        images = set(manifest_data.get('images', set()))
+        rules = list(manifest_data.get('rules', list()))
+
+    else:
+        # NOTE(mnaser): This list must be maintained manually because the image
+        #               registry must be able to support a few different versions
+        #               of Kubernetes since it is possible to have multiple
+        #               clusters running different versions of Kubernetes at the
+        #               same time.
+        images = set(
+            _get_all_kubeadm_images()
+            + _get_calico_images()
+            + _get_cloud_provider_images()
+            + _get_infra_images()
+        )
+        rules = list()
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=parallel) as executor:
         future_to_image = {
             executor.submit(
-                _mirror_image, image, repository, insecure, crane_path
+                _mirror_image, image, repository, rules, insecure, crane_path
             ): image
             for image in images
         }
@@ -111,9 +124,14 @@ def main(repository, parallel, insecure):
                 )
 
 
-def _mirror_image(image: str, repository: str, insecure: bool, crane_path: str):
+def _mirror_image(image: str, repository: str, rules: list, insecure: bool, crane_path: str):
     src = image
-    dst = image_utils.get_image(image, repository)
+    dst = image_utils.get_image(image, repository, rules)
+
+    # NOTE(jrosser): whilst crane will load the image data it will
+    #                not create the required metadata in the repository
+    #                if the image destination contains a digest.
+    dst = dst.split('@')[0]
 
     try:
         command = [crane_path]

magnum_cluster_api/image_utils.py

@@ -13,7 +13,7 @@
 # under the License.
 
 import itertools
-
+import re
 import yaml
 
 
@@ -48,14 +48,38 @@ def update_manifest_images(cluster_uuid: str, file, repository=None, replacement
     return yaml.safe_dump_all(docs, default_flow_style=False)
 
 
-def get_image(name: str, repository: str = None):
+def get_image_with_rules(name: str, repository: str, rules: list):
+    """
+    Get the image name from the target registry applying substitution rules
+    """
+
+    for rule in rules:
+        pattern = rule.get('pattern')
+        replace = rule.get('replace')
+        if not pattern:
+            continue
+        if not replace:
+            continue
+
+        name, subs_made = re.subn(pattern, replace, name)
+
+        if subs_made:
+          break
+
+    name = name.format(repository=repository)
+    return name
+
+def get_image(name: str, repository: str = None, rules: list = []):
     """
     Get the image name from the target registry given a full image name.
     """
 
     if not repository:
         return name
 
+    if rules:
+        return get_image_with_rules(name, repository, rules)
+
     new_image_name = name
     if name.startswith("docker.io/calico"):
         new_image_name = name.replace("docker.io/calico/", f"{repository}/calico/")