Add os/layers size/arch fields to manifest model

closes: pulp#1767

CHANGES/1767.feature

@@ -0,0 +1 @@
+Added `architecture`, `os`, and `compressed_layers_size` fields to Manifest.

pulp_container/app/management/commands/container-handle-image-data.py

@@ -1,3 +1,4 @@
+import json
 from json.decoder import JSONDecodeError
 
 from gettext import gettext as _
@@ -42,7 +43,11 @@ def handle(self, *args, **options):
         manifests_v1 = Manifest.objects.filter(data__isnull=True, media_type=MEDIA_TYPE.MANIFEST_V1)
         manifests_updated_count += self.update_manifests(manifests_v1)
 
-        manifests_v2 = Manifest.objects.filter(Q(data__isnull=True) | Q(annotations={}, labels={}))
+        manifests_v2 = Manifest.objects.filter(
+            Q(data__isnull=True)
+            | Q(annotations={}, labels={})
+            | Q(architecture__isnull=True, os__isnull=True, compressed_layers_size__isnull=True)
+        )
         manifests_v2 = manifests_v2.exclude(
             media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_V1]
         )
@@ -68,6 +73,17 @@ def handle(self, *args, **options):
     def update_manifests(self, manifests_qs):
         manifests_updated_count = 0
         manifests_to_update = []
+        fields_to_update = [
+            "annotations",
+            "labels",
+            "is_bootable",
+            "is_flatpak",
+            "data",
+            "os",
+            "architecture",
+            "compressed_layers_size",
+        ]
+
         for manifest in manifests_qs.iterator():
             # suppress non-existing/already migrated artifacts and corrupted JSON files
             with suppress(ObjectDoesNotExist, JSONDecodeError):
@@ -76,7 +92,6 @@ def update_manifests(self, manifests_qs):
                     manifests_to_update.append(manifest)
 
             if len(manifests_to_update) > 1000:
-                fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
                 manifests_qs.model.objects.bulk_update(
                     manifests_to_update,
                     fields_to_update,
@@ -85,7 +100,6 @@ def update_manifests(self, manifests_qs):
                 manifests_to_update.clear()
 
         if manifests_to_update:
-            fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
             manifests_qs.model.objects.bulk_update(
                 manifests_to_update,
                 fields_to_update,
@@ -100,11 +114,24 @@ def init_manifest(self, manifest):
             manifest_data, raw_bytes_data = get_content_data(manifest_artifact)
             manifest.data = raw_bytes_data.decode("utf-8")
 
-            if not (manifest.annotations or manifest.labels):
+            if not (
+                manifest.annotations
+                or manifest.labels
+                or manifest.architecture
+                or manifest.os
+                or manifest.compressed_layers_size
+            ):
                 manifest.init_metadata(manifest_data)
 
             manifest._artifacts.clear()
 
             return True
+        elif manifest.media_type not in [MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] and not (
+            manifest.architecture or manifest.os or manifest.compressed_layers_size
+        ):
+            manifest_data = json.loads(manifest.data)
+            manifest.init_architecture_and_os(manifest_data)
+            manifest.init_compressed_layers_size(manifest_data)
+            return True
 
         return False

pulp_container/app/migrations/0042_update_manifest_and_manifestlistmanifest.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.16 on 2024-10-02 12:04
+import warnings
+
+from django.db import migrations, models
+
+def print_warning_for_updating_manifest_fields(apps, schema_editor):
+    warnings.warn(
+        "Run 'pulpcore-manager container-handle-image-data' to update the manifests' "
+        "os, architecture, and compressed_layers_size fields."
+    )
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0041_add_pull_through_pull_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='architecture',
+            field=models.TextField(null=True),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='compressed_layers_size',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='os',
+            field=models.TextField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='manifestlistmanifest',
+            name='architecture',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AlterField(
+            model_name='manifestlistmanifest',
+            name='os',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.RunPython(
+            print_warning_for_updating_manifest_fields,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+    ]

pulp_container/app/models.py

@@ -77,6 +77,11 @@ class Manifest(Content):
         labels (models.JSONField): Metadata stored inside the image configuration.
         is_bootable (models.BooleanField): Indicates whether the image is bootable or not.
         is_flatpak (models.BooleanField): Indicates whether the image is a flatpak package or not.
+        architecture (models.TextField): CPU architecture for which the binaries in the image are
+            designed to run.
+        os (models.TextField): Operating System which the image is built to run on.
+        compressed_layers_size (models.IntegerField): Sum of the sizes, in bytes, of all compressed
+            layers.
 
     Relations:
         blobs (models.ManyToManyField): Many-to-many relationship with Blob.
@@ -103,6 +108,9 @@ class Manifest(Content):
 
     annotations = models.JSONField(default=dict)
     labels = models.JSONField(default=dict)
+    architecture = models.TextField(null=True)
+    os = models.TextField(null=True)
+    compressed_layers_size = models.IntegerField(null=True)
 
     is_bootable = models.BooleanField(default=False)
     is_flatpak = models.BooleanField(default=False)
@@ -124,6 +132,8 @@ def init_metadata(self, manifest_data=None):
         has_annotations = self.init_annotations(manifest_data)
         has_labels = self.init_labels()
         has_image_nature = self.init_image_nature()
+        self.init_architecture_and_os(manifest_data)
+        self.init_compressed_layers_size(manifest_data)
         return has_annotations or has_labels or has_image_nature
 
     def init_annotations(self, manifest_data=None):
@@ -176,6 +186,31 @@ def init_manifest_nature(self):
         else:
             return False
 
+    def init_architecture_and_os(self, manifest_data):
+        # manifestv2 schema1 has the architecture definition in the Manifest (not in the ConfigBlob)
+        if architecture := manifest_data.get("architecture", None):
+            self.architecture = architecture
+            return
+
+        manifest_config = manifest_data.get("config", None)
+        config_blob_sha256 = manifest_config.get("digest", None)
+        blob_artifact = Artifact.objects.get(sha256=config_blob_sha256.removeprefix("sha256:"))
+        config_blob, _ = get_content_data(blob_artifact)
+        self.architecture = config_blob.get("architecture", None)
+        self.os = config_blob.get("os", None)
+
+    def init_compressed_layers_size(self, manifest_data):
+        # manifestv2 schema1 has only blobSum definition for each layer
+        if manifest_data.get("fsLayers", None):
+            self.compressed_layers_size = 0
+            return
+
+        layers = manifest_data.get("layers")
+        compressed_size = 0
+        for layer in layers:
+            compressed_size += layer.get("size")
+        self.compressed_layers_size = compressed_size
+
     def is_bootable_image(self):
         if (
             self.annotations.get("containers.bootc") == "1"
@@ -222,8 +257,9 @@ class ManifestListManifest(models.Model):
         manifest_list (models.ForeignKey): Many-to-one relationship with ManifestList.
     """
 
-    architecture = models.TextField()
-    os = models.TextField()
+    # in oci-index spec, platform is an optional field
+    architecture = models.TextField(default="", blank=True)
+    os = models.TextField(default="", blank=True)
     os_version = models.TextField(default="", blank=True)
     os_features = models.TextField(default="", blank=True)
     features = models.TextField(default="", blank=True)

pulp_container/app/registry.py

@@ -455,9 +455,12 @@ async def init_pending_content(self, digest, manifest_data, media_type, raw_text
             data=raw_text_data,
         )
 
-        # skip if media_type of schema1
+        # if media_type of schema1 configure only manifest architecture and compressed layers size
         if media_type in (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI):
             await sync_to_async(manifest.init_metadata)(manifest_data=manifest_data)
+        else:
+            await sync_to_async(manifest.init_architecture_and_os)(manifest_data=manifest_data)
+            await sync_to_async(manifest.init_compressed_layers_size)(manifest_data=manifest_data)
 
         try:
             await manifest.asave()

pulp_container/app/serializers.py

@@ -103,6 +103,21 @@ class ManifestSerializer(NoArtifactContentSerializer):
         default=False,
         help_text=_("A boolean determining whether the image bundles a Flatpak application"),
     )
+    architecture = serializers.CharField(
+        help_text="The CPU architecture which the binaries in this image are built to run on.",
+        required=False,
+        default=None,
+    )
+    os = serializers.CharField(
+        help_text="The name of the operating system which the image is built to run on.",
+        required=False,
+        default=None,
+    )
+    compressed_layers_size = serializers.IntegerField(
+        help_text="Specifies the sum of the sizes, in bytes, of all compressed layers",
+        required=False,
+        default=None,
+    )
 
     class Meta:
         fields = NoArtifactContentSerializer.Meta.fields + (
@@ -116,6 +131,9 @@ class Meta:
             "labels",
             "is_bootable",
             "is_flatpak",
+            "architecture",
+            "os",
+            "compressed_layers_size",
         )
         model = models.Manifest
 

pulp_container/app/tasks/builder.py

@@ -13,7 +13,7 @@
     Tag,
 )
 from pulp_container.constants import MEDIA_TYPE
-from pulp_container.app.utils import calculate_digest
+from pulp_container.app.utils import calculate_digest, get_content_data
 from pulpcore.plugin.models import Artifact, ContentArtifact, Content
 
 
@@ -83,11 +83,18 @@ def add_image_from_directory_to_repository(path, repository, tag):
 
         config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
         manifest.config_blob = config_blob
-        manifest.save()
+        blob_artifact = Artifact.objects.get(sha256=config_blob.digest.removeprefix("sha256:"))
+        config_blob_dict, _ = get_content_data(blob_artifact)
+        manifest.architecture = config_blob_dict.get("architecture", None)
+        manifest.os = config_blob_dict.get("os", None)
 
         pks_to_add = []
+        compressed_size = 0
         for layer in manifest_json["layers"]:
+            compressed_size += layer.get("size")
             pks_to_add.append(get_or_create_blob(layer, manifest, path).pk)
+        manifest.compressed_layers_size = compressed_size
+        manifest.save()
 
         pks_to_add.extend([manifest.pk, tag.pk, config_blob.pk])
         new_repo_version.add_content(Content.objects.filter(pk__in=pks_to_add))

pulp_container/app/tasks/sync_stages.py

@@ -332,12 +332,15 @@ async def handle_blobs(self, manifest_dc, content_data):
         Handle blobs.
         """
         manifest_dc.extra_data["blob_dcs"] = []
+        compressed_size = 0
         for layer in content_data.get("layers") or content_data.get("fsLayers"):
             if not self._include_layer(layer):
                 continue
+            compressed_size += layer.get("size", 0)
             blob_dc = self.create_blob(layer)
             manifest_dc.extra_data["blob_dcs"].append(blob_dc)
             await self.put(blob_dc)
+        manifest_dc.content.compressed_layers_size = compressed_size
         layer = content_data.get("config", None)
         if layer:
             blob_dc = self.create_blob(layer, deferred_download=False)
@@ -390,8 +393,8 @@ def create_manifest(self, manifest_data, raw_text_data, media_type, digest=None)
             media_type=media_type,
             data=raw_text_data,
             annotations=manifest_data.get("annotations", {}),
+            architecture=manifest_data.get("architecture", None),
         )
-
         manifest_dc = DeclarativeContent(content=manifest)
         return manifest_dc
 
@@ -470,13 +473,14 @@ async def create_listed_manifest(self, manifest_data):
             )
 
         platform = {}
-        p = manifest_data["platform"]
-        platform["architecture"] = p["architecture"]
-        platform["os"] = p["os"]
-        platform["features"] = p.get("features", "")
-        platform["variant"] = p.get("variant", "")
-        platform["os.version"] = p.get("os.version", "")
-        platform["os.features"] = p.get("os.features", "")
+        # in oci-index spec, platform is an optional field
+        if p := manifest_data.get("platform", None):
+            platform["architecture"] = p["architecture"]
+            platform["os"] = p["os"]
+            platform["features"] = p.get("features", "")
+            platform["variant"] = p.get("variant", "")
+            platform["os.version"] = p.get("os.version", "")
+            platform["os.features"] = p.get("os.features", "")
         man_dc = DeclarativeContent(content=manifest)
         return {"manifest_dc": man_dc, "platform": platform, "content_data": content_data}
 
@@ -626,19 +630,31 @@ def _post_save(self, batch):
                 manifest_lists.append(dc.content)
                 for listed_manifest in dc.extra_data["listed_manifests"]:
                     manifest_dc = listed_manifest["manifest_dc"]
-                    platform = listed_manifest["platform"]
-                    manifest_list_manifests.append(
-                        ManifestListManifest(
-                            manifest_list=manifest_dc.content,
-                            image_manifest=dc.content,
-                            architecture=platform["architecture"],
-                            os=platform["os"],
-                            features=platform.get("features"),
-                            variant=platform.get("variant"),
-                            os_version=platform.get("os.version"),
-                            os_features=platform.get("os.features"),
-                        )
+                    manifest_list_manifest = ManifestListManifest(
+                        manifest_list=manifest_dc.content,
+                        image_manifest=dc.content,
                     )
+                    if platform := listed_manifest.get("platform"):
+                        manifest_list_manifest.architecture = platform["architecture"]
+                        manifest_list_manifest.os = platform["os"]
+                        manifest_list_manifest.features = platform.get("features")
+                        manifest_list_manifest.variant = platform.get("variant")
+                        manifest_list_manifest.os_version = platform.get("os.version")
+                        manifest_list_manifest.os_features = platform.get("os.features")
+                    manifest_list_manifests.append(manifest_list_manifest)
+                continue
+
+            if "config_blob_dc" in dc.extra_data:
+                manifest_dc = dc.content
+                config_blob_sha256 = dc.extra_data["config_blob_dc"].content.digest
+                blob_artifact = Artifact.objects.get(
+                    sha256=config_blob_sha256.removeprefix("sha256:")
+                )
+                config_blob, _ = get_content_data(blob_artifact)
+                manifest_dc.architecture = config_blob.get("architecture", None)
+                manifest_dc.os = config_blob.get("os", None)
+                manifest_dc.save()
+
         if blob_manifests:
             BlobManifest.objects.bulk_create(blob_manifests, ignore_conflicts=True)
         if manifest_list_manifests:

pulp_container/tests/functional/api/test_build_images.py

@@ -31,6 +31,7 @@ def containerfile_name():
 
 def test_build_image(
     pulpcore_bindings,
+    container_manifest_api,
     container_repository_api,
     container_distribution_api,
     gen_object_with_cleanup,
@@ -61,3 +62,9 @@ def test_build_image(
     local_registry.pull(distribution.base_path)
     image = local_registry.inspect(distribution.base_path)
     assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"]
+
+    manifest = container_manifest_api.list(digest=image[0]["Digest"])
+    manifest = manifest.to_dict()["results"][0]
+    assert manifest["architecture"] == "amd64"
+    assert manifest["os"] == "linux"
+    assert manifest["compressed_layers_size"] > 0