From a75f663a9d1e7a1529a59d8f8a6467c638dd9637 Mon Sep 17 00:00:00 2001 From: Moustafa Moustafa Date: Thu, 19 Dec 2024 15:52:34 -0600 Subject: [PATCH 1/3] Add snapshot support --- ...tribution_snapshot_publication_snapshot.py | 23 +++++++ pulpcore/app/models/publication.py | 8 ++- pulpcore/app/serializers/publication.py | 5 +- pulpcore/content/handler.py | 68 ++++++++++++++++++- 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py diff --git a/pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py b/pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py new file mode 100644 index 0000000000..1539773a3a --- /dev/null +++ b/pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.17 on 2024-12-13 15:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0125_openpgpdistribution_openpgpkeyring_openpgppublickey_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="distribution", + name="snapshot", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="publication", + name="snapshot", + field=models.BooleanField(default=False), + ), + ] diff --git a/pulpcore/app/models/publication.py b/pulpcore/app/models/publication.py index c19d5cf032..9aae3596b6 100644 --- a/pulpcore/app/models/publication.py +++ b/pulpcore/app/models/publication.py @@ -98,12 +98,13 @@ class Publication(MasterModel): complete = models.BooleanField(db_index=True, default=False) pass_through = models.BooleanField(default=False) + snapshot = models.BooleanField(default=False) repository_version = models.ForeignKey("RepositoryVersion", on_delete=models.CASCADE) pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) @classmethod - def create(cls, repository_version, pass_through=False): + def create(cls, repository_version, pass_through=False, snapshot=False): """ Create a publication. @@ -125,7 +126,9 @@ def create(cls, repository_version, pass_through=False): Adds a Task.created_resource for the publication. """ with transaction.atomic(): - publication = cls(pass_through=pass_through, repository_version=repository_version) + publication = cls( + pass_through=pass_through, repository_version=repository_version, snapshot=snapshot + ) publication.save() resource = CreatedResource(content_object=publication) resource.save() @@ -649,6 +652,7 @@ class Distribution(MasterModel): base_path = models.TextField() pulp_domain = models.ForeignKey("Domain", default=get_domain_pk, on_delete=models.PROTECT) hidden = models.BooleanField(default=False, null=True) + snapshot = models.BooleanField(default=False) content_guard = models.ForeignKey(ContentGuard, null=True, on_delete=models.SET_NULL) publication = models.ForeignKey(Publication, null=True, on_delete=models.SET_NULL) diff --git a/pulpcore/app/serializers/publication.py b/pulpcore/app/serializers/publication.py index 738e969181..5cf47c397f 100644 --- a/pulpcore/app/serializers/publication.py +++ b/pulpcore/app/serializers/publication.py @@ -30,6 +30,7 @@ class PublicationSerializer(ModelSerializer): view_name_pattern=r"repositories(-.*/.*)?-detail", queryset=models.Repository.objects.all(), ) + snapshot = serializers.BooleanField(default=False) def validate(self, data): data = super().validate(data) @@ -62,7 +63,7 @@ def validate(self, data): class Meta: abstract = True model = models.Publication - fields = ModelSerializer.Meta.fields + ("repository_version", "repository") + fields = ModelSerializer.Meta.fields + ("repository_version", "repository", "snapshot") class ContentGuardSerializer(ModelSerializer): @@ -238,6 +239,7 @@ class DistributionSerializer(ModelSerializer): "not changed. If equals to `null`, no guarantee is provided about content changes." ) ) + snapshot = serializers.BooleanField(default=False) class Meta: model = models.Distribution @@ -250,6 +252,7 @@ class Meta: "pulp_labels", "name", "repository", + "snapshot", ) def _validate_path_overlap(self, path): diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index e043b0b1c1..8e94b44b32 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -118,6 +118,29 @@ def __init__(self, path, distros): super().__init__(body=html, headers={"Content-Type": "text/html"}) +from datetime import datetime +class SnapshotListings(HTTPOk): + """ + Response for browsing through the snapshots of a specific snapshot distro. + + This is returned when visiting the base path of a snapshot distro. + """ + + def __init__(self, path, repo): + """Create the HTML response.""" + + snapshots = ( + Publication.objects.filter(repository_version__repository=repo, snapshot=True) + .order_by("pulp_created") + .values_list("pulp_created", flat=True) + .distinct() + ) + dates = {f"{datetime.strftime(s, '%Y%m%dT%H%M%SZ')}/": s for s in snapshots} + directory_list = dates.keys() + html = Handler.render_html(directory_list, dates=dates, path=path) + super().__init__(body=html, headers={"Content-Type": "text/html"}) + + class ArtifactNotFound(Exception): """ The artifact associated with a published-artifact does not exist. @@ -312,7 +335,7 @@ def _match_distribution(cls, path, add_trailing_slash=True): distro_model = cls.distribution_model or Distribution domain = get_domain() try: - return ( + distro_object = ( distro_model.objects.filter(pulp_domain=domain) .select_related( "repository", @@ -326,6 +349,49 @@ def _match_distribution(cls, path, add_trailing_slash=True): .get(base_path__in=base_paths) .cast() ) + + if distro_object.snapshot: + # Determine whether it's a listing or a specific snapshot + if path == f"{distro_object.base_path}/": + if not path_ends_in_slash: + raise HTTPMovedPermanently(f"/{path}") + raise SnapshotListings(path=path, repo=distro_object.repository) + else: + # Validate path i.e. //.* + base_path = distro_object.base_path + pattern = rf"^{re.escape(base_path)}/(\d{{8}}T\d{{6}}Z)(/.*)?$" + re.compile(pattern) + match = re.search(pattern, path) + if match: + timestamp_str = match.group(1) + timestamp = datetime.strptime(timestamp_str, "%Y%m%dT%H%M%SZ") + timestamp = timestamp.replace(microsecond=999999) + else: + raise PathNotResolved(original_path) + + # Find the latest snapshot publication before or at the timestamp + snapshot_publication = ( + Publication.objects.filter( + pulp_created__lte=timestamp, + repository_version__repository=distro_object.repository, + snapshot=True, + ) + .order_by("-pulp_created") + .first() + ) + distro_object.base_path = f"{base_path}/{timestamp_str}" + distro_object.repository = None + distro_object.publication = snapshot_publication + # Or if we want to redirect + redirect = False + pub_timestamp_str = datetime.strftime( + snapshot_publication.pulp_created, "%Y%m%dT%H%M%SZ" + ) + if redirect and pub_timestamp_str != timestamp_str: + raise HTTPMovedPermanently( + f"{settings.CONTENT_PATH_PREFIX}{base_path}/{pub_timestamp_str}/" + ) + return distro_object except ObjectDoesNotExist: if path.rstrip("/") in base_paths: distros = distro_model.objects.filter( From 51105431dcb098dddbd0ce25d6a2bf1789e31f4f Mon Sep 17 00:00:00 2001 From: Moustafa Moustafa Date: Fri, 20 Dec 2024 11:09:12 -0600 Subject: [PATCH 2/3] Add napshot support to file --- pulp_file/app/tasks/publishing.py | 6 ++++-- pulp_file/app/viewsets.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pulp_file/app/tasks/publishing.py b/pulp_file/app/tasks/publishing.py index 36e86b71a4..7c0f6628d7 100644 --- a/pulp_file/app/tasks/publishing.py +++ b/pulp_file/app/tasks/publishing.py @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) -def publish(manifest, repository_version_pk): +def publish(manifest, repository_version_pk, snapshot): """ Create a Publication based on a RepositoryVersion. @@ -37,7 +37,9 @@ def publish(manifest, repository_version_pk): ) with tempfile.TemporaryDirectory(dir="."): - with FilePublication.create(repo_version, pass_through=True) as publication: + with FilePublication.create( + repo_version, pass_through=True, snapshot=snapshot + ) as publication: publication.manifest = manifest if manifest: manifest = Manifest(manifest) diff --git a/pulp_file/app/viewsets.py b/pulp_file/app/viewsets.py index b2b80cac68..1a6674c940 100644 --- a/pulp_file/app/viewsets.py +++ b/pulp_file/app/viewsets.py @@ -433,11 +433,16 @@ def create(self, request): serializer.is_valid(raise_exception=True) repository_version = serializer.validated_data.get("repository_version") manifest = serializer.validated_data.get("manifest") + snapshot = serializer.validated_data.get("snapshot") result = dispatch( tasks.publish, shared_resources=[repository_version.repository], - kwargs={"repository_version_pk": str(repository_version.pk), "manifest": manifest}, + kwargs={ + "repository_version_pk": str(repository_version.pk), + "manifest": manifest, + "snapshot": snapshot, + }, ) return OperationPostponedResponse(result, request) From 70bf9ad19106946ea67841cf6a61f740a970b061 Mon Sep 17 00:00:00 2001 From: Moustafa Moustafa Date: Fri, 20 Dec 2024 12:54:48 -0600 Subject: [PATCH 3/3] update migrations --- ...ot.py => 0127_distribution_snapshot_publication_snapshot.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pulpcore/app/migrations/{0126_distribution_snapshot_publication_snapshot.py => 0127_distribution_snapshot_publication_snapshot.py} (85%) diff --git a/pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py b/pulpcore/app/migrations/0127_distribution_snapshot_publication_snapshot.py similarity index 85% rename from pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py rename to pulpcore/app/migrations/0127_distribution_snapshot_publication_snapshot.py index 1539773a3a..22b4b2d275 100644 --- a/pulpcore/app/migrations/0126_distribution_snapshot_publication_snapshot.py +++ b/pulpcore/app/migrations/0127_distribution_snapshot_publication_snapshot.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("core", "0125_openpgpdistribution_openpgpkeyring_openpgppublickey_and_more"), + ("core", "0126_remoteartifact_failed_at"), ] operations = [