Merge pull request #1 from Moustafa-Moustafa/snapshot-core-poc

Add snapshot support

pulp_file/app/tasks/publishing.py

@@ -19,7 +19,7 @@
 log = logging.getLogger(__name__)
 
 
-def publish(manifest, repository_version_pk):
+def publish(manifest, repository_version_pk, snapshot):
     """
     Create a Publication based on a RepositoryVersion.
 
@@ -37,7 +37,9 @@ def publish(manifest, repository_version_pk):
     )
 
     with tempfile.TemporaryDirectory(dir="."):
-        with FilePublication.create(repo_version, pass_through=True) as publication:
+        with FilePublication.create(
+            repo_version, pass_through=True, snapshot=snapshot
+        ) as publication:
             publication.manifest = manifest
             if manifest:
                 manifest = Manifest(manifest)

pulp_file/app/viewsets.py

@@ -433,11 +433,16 @@ def create(self, request):
         serializer.is_valid(raise_exception=True)
         repository_version = serializer.validated_data.get("repository_version")
         manifest = serializer.validated_data.get("manifest")
+        snapshot = serializer.validated_data.get("snapshot")
 
         result = dispatch(
             tasks.publish,
             shared_resources=[repository_version.repository],
-            kwargs={"repository_version_pk": str(repository_version.pk), "manifest": manifest},
+            kwargs={
+                "repository_version_pk": str(repository_version.pk),
+                "manifest": manifest,
+                "snapshot": snapshot,
+            },
         )
         return OperationPostponedResponse(result, request)
 

pulpcore/app/migrations/0127_distribution_snapshot_publication_snapshot.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.17 on 2024-12-13 15:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0126_remoteartifact_failed_at"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="distribution",
+            name="snapshot",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="publication",
+            name="snapshot",
+            field=models.BooleanField(default=False),
+        ),
+    ]

pulpcore/app/models/publication.py

@@ -98,12 +98,13 @@ class Publication(MasterModel):
 
     complete = models.BooleanField(db_index=True, default=False)
     pass_through = models.BooleanField(default=False)
+    snapshot = models.BooleanField(default=False)
 
     repository_version = models.ForeignKey("RepositoryVersion", on_delete=models.CASCADE)
     pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
 
     @classmethod
-    def create(cls, repository_version, pass_through=False):
+    def create(cls, repository_version, pass_through=False, snapshot=False):
         """
         Create a publication.
 
@@ -125,7 +126,9 @@ def create(cls, repository_version, pass_through=False):
             Adds a Task.created_resource for the publication.
         """
         with transaction.atomic():
-            publication = cls(pass_through=pass_through, repository_version=repository_version)
+            publication = cls(
+                pass_through=pass_through, repository_version=repository_version, snapshot=snapshot
+            )
             publication.save()
             resource = CreatedResource(content_object=publication)
             resource.save()
@@ -649,6 +652,7 @@ class Distribution(MasterModel):
     base_path = models.TextField()
     pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT)
     hidden = models.BooleanField(default=False, null=True)
+    snapshot = models.BooleanField(default=False)
 
     content_guard = models.ForeignKey(ContentGuard, null=True, on_delete=models.SET_NULL)
     publication = models.ForeignKey(Publication, null=True, on_delete=models.SET_NULL)

pulpcore/app/serializers/publication.py

@@ -30,6 +30,7 @@ class PublicationSerializer(ModelSerializer):
         view_name_pattern=r"repositories(-.*/.*)?-detail",
         queryset=models.Repository.objects.all(),
     )
+    snapshot = serializers.BooleanField(default=False)
 
     def validate(self, data):
         data = super().validate(data)
@@ -62,7 +63,7 @@ def validate(self, data):
     class Meta:
         abstract = True
         model = models.Publication
-        fields = ModelSerializer.Meta.fields + ("repository_version", "repository")
+        fields = ModelSerializer.Meta.fields + ("repository_version", "repository", "snapshot")
 
 
 class ContentGuardSerializer(ModelSerializer):
@@ -238,6 +239,7 @@ class DistributionSerializer(ModelSerializer):
             "not changed. If equals to `null`, no guarantee is provided about content changes."
         )
     )
+    snapshot = serializers.BooleanField(default=False)
 
     class Meta:
         model = models.Distribution
@@ -250,6 +252,7 @@ class Meta:
             "pulp_labels",
             "name",
             "repository",
+            "snapshot",
         )
 
     def _validate_path_overlap(self, path):

pulpcore/content/handler.py

@@ -118,6 +118,29 @@ def __init__(self, path, distros):
         super().__init__(body=html, headers={"Content-Type": "text/html"})
 
 
+from datetime import datetime
+class SnapshotListings(HTTPOk):
+    """
+    Response for browsing through the snapshots of a specific snapshot distro.
+
+    This is returned when visiting the base path of a snapshot distro.
+    """
+
+    def __init__(self, path, repo):
+        """Create the HTML response."""
+
+        snapshots = (
+            Publication.objects.filter(repository_version__repository=repo, snapshot=True)
+            .order_by("pulp_created")
+            .values_list("pulp_created", flat=True)
+            .distinct()
+        )
+        dates = {f"{datetime.strftime(s, '%Y%m%dT%H%M%SZ')}/": s for s in snapshots}
+        directory_list = dates.keys()
+        html = Handler.render_html(directory_list, dates=dates, path=path)
+        super().__init__(body=html, headers={"Content-Type": "text/html"})
+
+
 class ArtifactNotFound(Exception):
     """
     The artifact associated with a published-artifact does not exist.
@@ -312,7 +335,7 @@ def _match_distribution(cls, path, add_trailing_slash=True):
         distro_model = cls.distribution_model or Distribution
         domain = get_domain()
         try:
-            return (
+            distro_object = (
                 distro_model.objects.filter(pulp_domain=domain)
                 .select_related(
                     "repository",
@@ -326,6 +349,49 @@ def _match_distribution(cls, path, add_trailing_slash=True):
                 .get(base_path__in=base_paths)
                 .cast()
             )
+
+            if distro_object.snapshot:
+                # Determine whether it's a listing or a specific snapshot
+                if path == f"{distro_object.base_path}/":
+                    if not path_ends_in_slash:
+                        raise HTTPMovedPermanently(f"/{path}")
+                    raise SnapshotListings(path=path, repo=distro_object.repository)
+                else:
+                    # Validate path i.e. <base_path>/<timestamp>/.*
+                    base_path = distro_object.base_path
+                    pattern = rf"^{re.escape(base_path)}/(\d{{8}}T\d{{6}}Z)(/.*)?$"
+                    re.compile(pattern)
+                    match = re.search(pattern, path)
+                    if match:
+                        timestamp_str = match.group(1)
+                        timestamp = datetime.strptime(timestamp_str, "%Y%m%dT%H%M%SZ")
+                        timestamp = timestamp.replace(microsecond=999999)
+                    else:
+                        raise PathNotResolved(original_path)
+
+                    # Find the latest snapshot publication before or at the timestamp
+                    snapshot_publication = (
+                        Publication.objects.filter(
+                            pulp_created__lte=timestamp,
+                            repository_version__repository=distro_object.repository,
+                            snapshot=True,
+                        )
+                        .order_by("-pulp_created")
+                        .first()
+                    )
+                    distro_object.base_path = f"{base_path}/{timestamp_str}"
+                    distro_object.repository = None
+                    distro_object.publication = snapshot_publication
+                    # Or if we want to redirect
+                    redirect = False
+                    pub_timestamp_str = datetime.strftime(
+                        snapshot_publication.pulp_created, "%Y%m%dT%H%M%SZ"
+                    )
+                    if redirect and pub_timestamp_str != timestamp_str:
+                        raise HTTPMovedPermanently(
+                            f"{settings.CONTENT_PATH_PREFIX}{base_path}/{pub_timestamp_str}/"
+                        )
+            return distro_object
         except ObjectDoesNotExist:
             if path.rstrip("/") in base_paths:
                 distros = distro_model.objects.filter(