From 6cc99dddd7f729411bfaf8c430b2f131db996e27 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 11 Dec 2024 13:29:07 +0200 Subject: [PATCH 1/3] Moved django-images module into PA repo --- docs/BACKGROUND.md | 1 - pombola/core/admin.py | 2 +- pombola/core/images/__init__.py | 0 pombola/core/images/admin.py | 34 ++++++ .../core/images/migrations/0001_initial.py | 26 +++++ .../migrations/0002_auto_20160323_1424.py | 20 ++++ pombola/core/images/migrations/__init__.py | 0 pombola/core/images/models.py | 71 ++++++++++++ pombola/core/images/tests.py | 104 ++++++++++++++++++ pombola/core/images/tests/bar.png | Bin 0 -> 357 bytes pombola/core/images/tests/baz.png | Bin 0 -> 393 bytes pombola/core/images/tests/foo.png | Bin 0 -> 402 bytes pombola/core/images/urls.py | 1 + pombola/core/images/views.py | 1 + .../import_member_photos.py | 2 +- pombola/core/management/merge.py | 2 +- pombola/core/migrations/0001_initial.py | 6 +- pombola/core/models.py | 2 +- pombola/core/tests/test_popolo.py | 2 +- pombola/settings/base.py | 1 - ...mport_new_constituency_office_locations.py | 2 +- requirements.txt | 1 - 22 files changed, 266 insertions(+), 12 deletions(-) create mode 100644 pombola/core/images/__init__.py create mode 100644 pombola/core/images/admin.py create mode 100644 pombola/core/images/migrations/0001_initial.py create mode 100644 pombola/core/images/migrations/0002_auto_20160323_1424.py create mode 100644 pombola/core/images/migrations/__init__.py create mode 100644 pombola/core/images/models.py create mode 100644 pombola/core/images/tests.py create mode 100644 pombola/core/images/tests/bar.png create mode 100644 pombola/core/images/tests/baz.png create mode 100644 pombola/core/images/tests/foo.png create mode 100644 pombola/core/images/urls.py create mode 100644 pombola/core/images/views.py diff --git a/docs/BACKGROUND.md b/docs/BACKGROUND.md index 99e427191..24388f224 100644 --- a/docs/BACKGROUND.md +++ b/docs/BACKGROUND.md @@ -79,7 +79,6 @@ into packages that can be installed from PyPI. This is because: Some examples of this are: -* https://github.com/mysociety/django-images * https://github.com/mysociety/django-info-pages * https://github.com/mysociety/django-slug-helpers diff --git a/pombola/core/admin.py b/pombola/core/admin.py index 1ddc80d35..536a4decd 100644 --- a/pombola/core/admin.py +++ b/pombola/core/admin.py @@ -8,7 +8,7 @@ from ajax_select import make_ajax_form from ajax_select.admin import AjaxSelectAdmin -from images.admin import ImageAdminInline +from pombola.core.images.admin import ImageAdminInline from slug_helpers.admin import StricterSlugFieldMixin from pombola.core import models diff --git a/pombola/core/images/__init__.py b/pombola/core/images/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pombola/core/images/admin.py b/pombola/core/images/admin.py new file mode 100644 index 000000000..9f5054ac0 --- /dev/null +++ b/pombola/core/images/admin.py @@ -0,0 +1,34 @@ +from django.contrib import admin +try: + from django.contrib.contenttypes.admin import GenericTabularInline +except ImportError: + # This fallback import is the version that was deprecated in + # Django 1.7 and is removed in 1.9: + from django.contrib.contenttypes.generic import GenericTabularInline + + +from sorl.thumbnail import get_thumbnail +from sorl.thumbnail.admin import AdminImageMixin + +from pombola.core.images import models + + +class ImageAdmin(AdminImageMixin, admin.ModelAdmin): + list_display = [ 'thumbnail', 'content_object', 'is_primary', 'source', ] + search_fields = ['person__legal_name', 'id', 'source'] + + def thumbnail(self, obj): + if obj.image: + im = get_thumbnail(obj.image, '100x100') + return '' % ( im.url ) + else: + return "NO IMAGE FOUND" + thumbnail.allow_tags = True + + +class ImageAdminInline(AdminImageMixin, GenericTabularInline): + model = models.Image + extra = 0 + can_delete = True + +admin.site.register( models.Image, ImageAdmin ) diff --git a/pombola/core/images/migrations/0001_initial.py b/pombola/core/images/migrations/0001_initial.py new file mode 100644 index 000000000..eaf6cd74f --- /dev/null +++ b/pombola/core/images/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import sorl.thumbnail.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Image', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('object_id', models.PositiveIntegerField()), + ('image', sorl.thumbnail.fields.ImageField(upload_to=b'images')), + ('source', models.CharField(max_length=400)), + ('is_primary', models.BooleanField(default=False)), + ('content_type', models.ForeignKey(to='contenttypes.ContentType')), + ], + ), + ] diff --git a/pombola/core/images/migrations/0002_auto_20160323_1424.py b/pombola/core/images/migrations/0002_auto_20160323_1424.py new file mode 100644 index 000000000..2059faca9 --- /dev/null +++ b/pombola/core/images/migrations/0002_auto_20160323_1424.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import sorl.thumbnail.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('images', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(max_length=512, upload_to=b'images'), + ), + ] diff --git a/pombola/core/images/migrations/__init__.py b/pombola/core/images/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pombola/core/images/models.py b/pombola/core/images/models.py new file mode 100644 index 000000000..99b8c7333 --- /dev/null +++ b/pombola/core/images/models.py @@ -0,0 +1,71 @@ +from django.db import models +from django.contrib.contenttypes.models import ContentType +try: + from django.contrib.contenttypes.fields import GenericForeignKey +except ImportError: + # This fallback import is the version that was deprecated in + # Django 1.7 and is removed in 1.9: + from django.contrib.contenttypes.generic import GenericForeignKey + +from sorl.thumbnail import ImageField + + +class Image(models.Model): + + class Meta: + db_table = 'images_image' + + # link to other objects using the ContentType system + content_type = models.ForeignKey(ContentType) + object_id = models.PositiveIntegerField() + content_object = GenericForeignKey('content_type', 'object_id') + + # store the actual image + image = ImageField(upload_to="images", max_length=512) + + # added + source = models.CharField(max_length=400) + # user + + is_primary = models.BooleanField( default=False ) + + def save(self, *args, **kwargs): + """ + Only one image should be marked as is_primary for an object. + """ + + # other images for this object + siblings = Image.objects.filter( + content_type = self.content_type, + object_id = self.object_id, + ) + + # check that we are not first entry for content_object + if not siblings.count(): + self.is_primary = True + + super(Image, self).save(*args, **kwargs) + + # If we are true then make sure all others are false + if self.is_primary is True: + + primary_siblings = siblings.exclude( is_primary = False ).exclude( id = self.id ) + + for sibling in primary_siblings: + sibling.is_primary = False + sibling.save() + + +class HasImageMixin(object): + + def primary_image(self): + primary_image_model = self.primary_image_model() + if primary_image_model: + return primary_image_model.image + return None + + def primary_image_model(self): + primary_images = [i for i in self.images.all() if i.is_primary] + if primary_images: + return primary_images[0] + return None diff --git a/pombola/core/images/tests.py b/pombola/core/images/tests.py new file mode 100644 index 000000000..e59d478b3 --- /dev/null +++ b/pombola/core/images/tests.py @@ -0,0 +1,104 @@ +import os + +from django.test import TestCase +from django.core.files import File +from django.contrib.sites.models import Site +from django.contrib.contenttypes.models import ContentType + +from .models import Image + +from sorl.thumbnail import get_thumbnail + +class ImageTest(TestCase): + + def get_example_file_content(self, filename): + """ + Open the given file and return it + """ + full_path = os.path.join( + os.path.abspath( os.path.dirname(__file__) ), + 'tests', + filename, + ) + + return File(open(full_path, 'rb')) + + def reload_image(self, image): + """because there is no reload in Django models (silly really)""" + return Image.objects.get(id=image.id) + + def test_uploading(self): + """ + Test that uploading an image works + """ + + test_site = Site.objects.all()[0] + test_site_ct = ContentType.objects.get_for_model(test_site) + test_site_id = test_site.id + + # check that there are no images + self.assertEqual( Image.objects.all().count(), 0 ) + + # create an image + first = Image( + content_type = test_site_ct, + object_id = test_site_id, + source = 'test directory', + ) + first.image.save( + name = 'foo.png', + content = self.get_example_file_content('foo.png'), + ) + + # check that the is_primary is true + self.assertTrue( first.is_primary ) + + # create another image + second = Image( + content_type = test_site_ct, + object_id = test_site_id, + source = 'test directory', + ) + second.image.save( + name = 'bar.png', + content = self.get_example_file_content('bar.png'), + ) + + # check that is_primary is false + self.assertTrue( self.reload_image(first).is_primary ) + self.assertFalse( self.reload_image(second).is_primary ) + + # change is_primary on second image + second.is_primary = True + second.save() + + # check that it changed on first too + self.assertFalse( self.reload_image(first).is_primary ) + self.assertTrue( self.reload_image(second).is_primary ) + + # create a third image with is_primary true at the start + third = Image( + content_type = test_site_ct, + object_id = test_site_id, + source = 'test directory', + is_primary = True, + ) + third.image.save( + name = 'baz.png', + content = self.get_example_file_content('baz.png'), + ) + + # check that is_primary is updated for all + self.assertFalse( self.reload_image(first).is_primary ) + self.assertFalse( self.reload_image(second).is_primary ) + self.assertTrue( self.reload_image(third).is_primary ) + + # Now try to create an thumbnail with sorl. If this fails + # with "IOError: decoder zip not available", then probably + # this is a problem with an old version of PIL, or one that + # wasn't installed when the right build dependencies were + # present. The simplest solution in most solutions is: + # pip uninstall PIL + # pip install pillow + + get_thumbnail(third.image, '100x100', crop='center', quality=99) diff --git a/pombola/core/images/tests/bar.png b/pombola/core/images/tests/bar.png new file mode 100644 index 0000000000000000000000000000000000000000..3a60f3bf3a3139fc365cb6e5d09970b75e14eddd GIT binary patch literal 357 zcmV-r0h<1aP)=&1kz009L_L_t(YiS3dxPs1<}g{{O0)>o*> z7`0`uY0FF{21X*p5AeoV1L};0`oFp7WPkzdvZA~pzDW1U=l9N-Nt6Dcs>Nu;qn(hk zh~3@h(r#Y1JQwu6;JwE(00*faA(~4Oow*6uB_v5_lPl^#qy~xzK9-4{{=&1kz00AjUL_t(YiS3d-Ps1<}hE1H%j9j5f zMz=z+f-0b6lSrszQ3R|t`2%!7tYqMSbFXVqI@nEDp>x6Q7q2>FB%$n+<=myD)}Cz)`@zWJOWv%A nTDZK*HW`3Gm({)v3hL_z0s$|Eyp3iq00000NkvXXu0mjfl&Y_Y literal 0 HcmV?d00001 diff --git a/pombola/core/images/tests/foo.png b/pombola/core/images/tests/foo.png new file mode 100644 index 0000000000000000000000000000000000000000..feb0867f804d084e2af2d1676cd344af7bff833c GIT binary patch literal 402 zcmV;D0d4+?P)!93-~Q({h4v9ST~MT4SvAwp5*a(?wjhsc7YU`^XY zNFGo0n_D*`fyYYWLx>X5@*MH^h^!UrdM5%*gz)*g1_{?>WwB<`FLfSGNY`;50j{0M zT9=xlhq>S>nmZ8_Q>?}~6xyAie!~fo6mc80$?6@y`9?#goB%80} wiapcpYd1bIH#Xu+)u!EOxmb212K@_t0(K%W$TF7?zW@LL07*qoM6N<$f*bX%g#Z8m literal 0 HcmV?d00001 diff --git a/pombola/core/images/urls.py b/pombola/core/images/urls.py new file mode 100644 index 000000000..16a46b99c --- /dev/null +++ b/pombola/core/images/urls.py @@ -0,0 +1 @@ +# This app doesn't provide any URLs diff --git a/pombola/core/images/views.py b/pombola/core/images/views.py new file mode 100644 index 000000000..60f00ef0e --- /dev/null +++ b/pombola/core/images/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/pombola/core/kenya_import_scripts/import_member_photos.py b/pombola/core/kenya_import_scripts/import_member_photos.py index 6471f742a..65273a297 100644 --- a/pombola/core/kenya_import_scripts/import_member_photos.py +++ b/pombola/core/kenya_import_scripts/import_member_photos.py @@ -17,7 +17,7 @@ from django.core.files.base import ContentFile from pombola.core import models -from pombola.images.models import Image +from pombola.core.images.models import Image constituency_kind = models.PlaceKind.objects.get(slug="constituency") diff --git a/pombola/core/management/merge.py b/pombola/core/management/merge.py index 9e2904756..39ed35320 100644 --- a/pombola/core/management/merge.py +++ b/pombola/core/management/merge.py @@ -10,7 +10,7 @@ from django.db import transaction from slug_helpers.models import SlugRedirect -from images.models import Image +from pombola.core.images.models import Image import pombola.core.models as core_models diff --git a/pombola/core/migrations/0001_initial.py b/pombola/core/migrations/0001_initial.py index 495e3f27c..a972063ae 100644 --- a/pombola/core/migrations/0001_initial.py +++ b/pombola/core/migrations/0001_initial.py @@ -7,7 +7,7 @@ import markitup.fields import django.contrib.gis.db.models.fields -import images.models +import pombola.core.images.models class Migration(migrations.Migration): @@ -118,7 +118,7 @@ class Migration(migrations.Migration): options={ 'ordering': ['slug'], }, - bases=(models.Model, images.models.HasImageMixin, pombola.core.models.IdentifierMixin), + bases=(models.Model, pombola.core.images.models.HasImageMixin, pombola.core.models.IdentifierMixin), ), migrations.CreateModel( name='OrganisationKind', @@ -209,7 +209,7 @@ class Migration(migrations.Migration): options={ 'ordering': ['sort_name'], }, - bases=(images.models.HasImageMixin, models.Model, pombola.core.models.IdentifierMixin), + bases=(pombola.core.images.models.HasImageMixin, models.Model, pombola.core.models.IdentifierMixin), ), migrations.CreateModel( name='Place', diff --git a/pombola/core/models.py b/pombola/core/models.py index a995516bc..433f05ce6 100644 --- a/pombola/core/models.py +++ b/pombola/core/models.py @@ -31,7 +31,7 @@ from django_date_extensions.fields import ApproximateDateField, ApproximateDate from slug_helpers.models import validate_slug_not_redirecting -from images.models import HasImageMixin, Image +from pombola.core.images.models import HasImageMixin, Image from pombola.tasks.models import Task diff --git a/pombola/core/tests/test_popolo.py b/pombola/core/tests/test_popolo.py index 38f84d388..96bd26e19 100644 --- a/pombola/core/tests/test_popolo.py +++ b/pombola/core/tests/test_popolo.py @@ -8,7 +8,7 @@ from mapit.models import Generation, Area, Type -from images.models import Image +from pombola.core.images.models import Image from pombola.core import models from pombola.core.popolo import get_popolo_data diff --git a/pombola/settings/base.py b/pombola/settings/base.py index 497fcfd1a..7ab2ef841 100644 --- a/pombola/settings/base.py +++ b/pombola/settings/base.py @@ -436,7 +436,6 @@ "pipeline", "mapit", "popolo", - "images", # easy_thumbnails is required by SayIt; it needs to be in # INSTALLED_APPS so that its table is created so that we can # create SayIt speakers. It should be after sorl.thumbnails so diff --git a/pombola/south_africa/management/commands/south_africa_import_new_constituency_office_locations.py b/pombola/south_africa/management/commands/south_africa_import_new_constituency_office_locations.py index 0a461bf40..b7ed25b0a 100644 --- a/pombola/south_africa/management/commands/south_africa_import_new_constituency_office_locations.py +++ b/pombola/south_africa/management/commands/south_africa_import_new_constituency_office_locations.py @@ -26,7 +26,7 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.gis.geos import Point -from images.models import Image +from pombola.core.images.models import Image from pombola.core.models import ( Organisation, diff --git a/requirements.txt b/requirements.txt index c69084060..7faf2e5cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -94,7 +94,6 @@ mysociety-django-sluggable==0.6.1.1 -e git+https://github.com/mysociety/popolo-name-resolver@a6fca27e080acdb475e6fd2e1382592b0c0a0fc5#egg=popolo-name-resolver mysociety-django-popolo==0.1.0 -mysociety-django-images==0.0.6 python-dateutil==2.4.2 From 2ba19738758f6409d1a5a416081694741b78f348 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 11 Dec 2024 14:17:53 +0200 Subject: [PATCH 2/3] Fixed migrations --- .../migrations/0017_auto_20241211_1411.py | 27 +++++++++++++++++++ pombola/core/models.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 pombola/core/migrations/0017_auto_20241211_1411.py diff --git a/pombola/core/migrations/0017_auto_20241211_1411.py b/pombola/core/migrations/0017_auto_20241211_1411.py new file mode 100644 index 000000000..7decbdd83 --- /dev/null +++ b/pombola/core/migrations/0017_auto_20241211_1411.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2024-12-11 14:11 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import sorl.thumbnail.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('core', '0016_auto_20201119_0713'), + ] + + operations = [ + migrations.AlterModelOptions( + name='organisationhistory', + options={'ordering': ['date_changed'], 'verbose_name_plural': 'organisation histories'}, + ), + migrations.AlterField( + model_name='organisationhistory', + name='new_organisation', + field=models.ForeignKey(default=0, on_delete=django.db.models.deletion.CASCADE, related_name='org_history_new', to='core.Organisation'), + ) + ] diff --git a/pombola/core/models.py b/pombola/core/models.py index 433f05ce6..1e6b5d9df 100644 --- a/pombola/core/models.py +++ b/pombola/core/models.py @@ -1972,7 +1972,7 @@ def raw_query_with_prefetch(query_model, query, params, fields_prefetches): class OrganisationHistory(ModelBase): old_organisation = models.ForeignKey('Organisation', null=False, related_name='org_history_old') - new_organisation = models.ForeignKey('Organisation', null=False, related_name='org_history_new') + new_organisation = models.ForeignKey('Organisation', null=False, related_name='org_history_new', default=0) date_changed = DateField(null=False) class Meta: From a1637ec456a2d2672ffaee9910d4c6684a1071bd Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 11 Dec 2024 18:05:39 +0200 Subject: [PATCH 3/3] Added S3 CloudFront Image files --- pombola/settings/base.py | 23 +++++++++++++++++------ requirements.txt | 3 +++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pombola/settings/base.py b/pombola/settings/base.py index 7ab2ef841..203f08254 100644 --- a/pombola/settings/base.py +++ b/pombola/settings/base.py @@ -142,11 +142,6 @@ # All uploaded files world-readable FILE_UPLOAD_PERMISSIONS = 0o644 # 'rw-r--r--' -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash. -# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = "/media_root/" - # Use django-pipeline for handling static files STATICFILES_STORAGE = "pipeline.storage.PipelineCachedStorage" @@ -460,12 +455,28 @@ "django_nose", "django_extensions", "rest_framework", - "djcelery" + "djcelery", ) + if os.environ.get("DJANGO_DEBUG_TOOLBAR", "true").lower() == "true": INSTALLED_APPS += ("debug_toolbar",) +if os.environ.get("AWS_STORAGE_BUCKET_NAME"): + STATIC_HOST = os.environ.get("STATIC_HOST", "static.pa.org.za") + INSTALLED_APPS += ("storages",) + AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_STORAGE_BUCKET_NAME") + AWS_S3_REGION_NAME = 'af-south-1' + AWS_S3_CUSTOM_DOMAIN = STATIC_HOST + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + MEDIA_URL = '//{}/'.format(AWS_S3_CUSTOM_DOMAIN) +else: + # URL that handles the media served from MEDIA_ROOT. Make sure to use a + # trailing slash. + # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" + MEDIA_URL = "/media_root/" + + def insert_after(sequence, existing_item, item_to_put_after): """A helper method for inserting an item after another in a sequence diff --git a/requirements.txt b/requirements.txt index 7faf2e5cf..28187241a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,8 @@ django-pipeline==1.6.13 django-pipeline-compass-rubygem==0.1.9 django-formtools==2.1 django-cors-headers==1.3.1 +django-storages==1.9.1 + ### API related djangorestframework==3.4.7 @@ -210,3 +212,4 @@ xlrd==1.2.0 #sentry-sdk[django]==1.39.2 sentry-sdk==0.14.3 +boto3==1.17.112 \ No newline at end of file