From 18617915263b56896186ab1583e6bbc10d665d26 Mon Sep 17 00:00:00 2001 From: J-E Castagnede Date: Thu, 11 Apr 2024 09:54:39 +0200 Subject: [PATCH] fix command --- georiviere/contribution/schema.py | 79 ++++++++++--------- .../river/management/commands/load_rivers.py | 19 +++-- georiviere/river/managers.py | 6 ++ georiviere/river/models.py | 20 +---- georiviere/utils/__init__.py | 0 georiviere/utils/mixins/managers.py | 7 ++ 6 files changed, 70 insertions(+), 61 deletions(-) create mode 100644 georiviere/river/managers.py create mode 100644 georiviere/utils/__init__.py create mode 100644 georiviere/utils/mixins/managers.py diff --git a/georiviere/contribution/schema.py b/georiviere/contribution/schema.py index 0c85d301..1315881c 100644 --- a/georiviere/contribution/schema.py +++ b/georiviere/contribution/schema.py @@ -7,9 +7,9 @@ InvasiveSpecies, HeritageSpecies, HeritageObservation, FishSpecies, NaturePollution, TypePollution) -# The json schema is summarize on : +# The json schema is summarized on : # https://github.com/Georiviere/Georiviere-admin/issues/139 -# Depending of the category and type of the contributions, some fields are available or not. +# Depending on the category and type of the contributions, some fields are available or not. # Here is the generation of the json schema used by the website portal. # The fields available depending on the type of contributions follow the documentation of jsonschema : # https://json-schema.org/understanding-json-schema/reference/conditionals.html @@ -18,37 +18,43 @@ def get_contribution_properties(): """ Feature properties as form initial data format (name / value) """ # TODO: Use directly field definition for type / title / max length - results = {'name_author': { - 'type': "string", - 'title': _("Name author"), - "maxLength": 128 - }, 'first_name_author': { - 'type': "string", - 'title': _("First name author"), - "maxLength": 128 - }, 'email_author': { - 'type': "string", - 'title': _("Email"), - 'format': "email" - }, 'date_observation': { - 'type': "string", - 'title': _("Observation's date"), - 'format': 'date' - }, 'description': { - 'type': "string", - 'title': _('Description') - }, 'category': { - "type": "string", - "title": _("Category"), - # TODO: Loop on contribution one to one field to get all possibilities - "enum": [ - str(ContributionQuantity._meta.verbose_name.title()), - str(ContributionQuality._meta.verbose_name.title()), - str(ContributionFaunaFlora._meta.verbose_name.title()), - str(ContributionLandscapeElements._meta.verbose_name.title()), - str(ContributionPotentialDamage._meta.verbose_name.title()) - ], - } + results = { + 'name_author': { + 'type': "string", + 'title': _("Name author"), + "maxLength": 128 + }, + 'first_name_author': { + 'type': "string", + 'title': _("First name author"), + "maxLength": 128 + }, + 'email_author': { + 'type': "string", + 'title': _("Email"), + 'format': "email" + }, + 'date_observation': { + 'type': "string", + 'title': _("Observation's date"), + 'format': 'date' + }, + 'description': { + 'type': "string", + 'title': _('Description') + }, + 'category': { + "type": "string", + "title": _("Category"), + # TODO: Loop on contribution one to one field to get all possibilities + "enum": [ + str(ContributionQuantity._meta.verbose_name.title()), + str(ContributionQuality._meta.verbose_name.title()), + str(ContributionFaunaFlora._meta.verbose_name.title()), + str(ContributionLandscapeElements._meta.verbose_name.title()), + str(ContributionPotentialDamage._meta.verbose_name.title()) + ], + } } if SeverityType.objects.exists(): results['severity'] = { @@ -110,8 +116,7 @@ def get_disruptive_jam(choices, meta): 'jam_type': { 'type': "string", - 'title': str(meta.get_field( - 'jam_type').related_model._meta.verbose_name.capitalize()), + 'title': str(meta.get_field('jam_type').related_model._meta.verbose_name.capitalize()), 'enum': list(JamType.objects.values_list('label', flat=True)) } }, @@ -125,7 +130,9 @@ def get_bank_erosion(choices, meta): 'if': { 'properties': { 'type': { - 'const': str(choices.BANK_EROSION.label)}} + 'const': str(choices.BANK_EROSION.label) + } + } }, 'then': { 'properties': { diff --git a/georiviere/river/management/commands/load_rivers.py b/georiviere/river/management/commands/load_rivers.py index 07b1acca..905c4983 100644 --- a/georiviere/river/management/commands/load_rivers.py +++ b/georiviere/river/management/commands/load_rivers.py @@ -15,8 +15,10 @@ def add_arguments(self, parser): parser.add_argument('file_path', help="File's path to import.") parser.add_argument('--name-attribute', '-n', action='store', dest='name', default='nom', help="Attribute name in file to use as river name") - parser.add_argument('--flush-streams', '-f', action='store_true', dest='flush', default=False, - help="Flush current streams") + parser.add_argument('--flush', '-f', action='store_true', dest='flush', default=False, + help="Flush rivers before import.") + parser.add_argument('--batch-size', '-bs', action='store', dest='batch_size', default=50, + help="Size of batch to use for bulk_create. Default is 50.") parser.add_argument('--default-name-attribute', '-nd', action='store', dest='default_name', default=_('River'), help="Default name to use if attribute name specified is empty") @@ -25,17 +27,17 @@ def handle(self, *args, **options): name_column = options.get('name') default_name = options.get('default_name') flush = options.get('flush') + batch_size = options.get('batch_size') data_source = DataSource(file_path) layer = data_source[0] total_count = len(layer) + self.stdout.write(f"Load rivers: {total_count} features to import") if flush: self.stdout.write("Delete streams.....", ending="") - Stream.objects.all().delete() + Stream.objects.truncate() self.stdout.write(self.style.SUCCESS("done!")) - batch_size = 50 - objs = (Stream(geom=feat.geom.geos, source_location=Point(feat.geom.geos[0]), name=feat.get(name_column) or default_name) for feat in layer) @@ -46,7 +48,10 @@ def handle(self, *args, **options): if not batch: break self.stdout.write(f"{count} / {total_count}", ending="") - Stream.objects.bulk_create(batch, batch_size) - self.stdout.write(self.style.SUCCESS(" ok!")) + try: + Stream.objects.bulk_create(batch, batch_size) + self.stdout.write(self.style.SUCCESS(" ok!")) + except Exception as e: + self.stdout.write(self.style.ERROR(" error!")) self.stdout.write(self.style.SUCCESS(f"Successfully import {total_count} rivers and associated morphologies / status")) diff --git a/georiviere/river/managers.py b/georiviere/river/managers.py new file mode 100644 index 00000000..b3060af7 --- /dev/null +++ b/georiviere/river/managers.py @@ -0,0 +1,6 @@ +from django.db import models +from georiviere.utils.mixins.managers import TruncateManagerMixin + + +class RiverManager(TruncateManagerMixin, models.Manager): + pass diff --git a/georiviere/river/models.py b/georiviere/river/models.py index 4aa38376..af65cd85 100644 --- a/georiviere/river/models.py +++ b/georiviere/river/models.py @@ -25,6 +25,7 @@ from georiviere.observations.models import Station from georiviere.proceeding.models import Proceeding from georiviere.maintenance.models import Intervention +from georiviere.river.managers import RiverManager from georiviere.studies.models import Study from georiviere.watershed.mixins import WatershedPropertiesMixin @@ -90,7 +91,7 @@ class FlowChoices(models.IntegerChoices): portals = models.ManyToManyField('portal.Portal', blank=True, related_name='streams', verbose_name=_("Published portals")) - + objects = RiverManager() capture_map_image_waitfor = '.other_object_enum_loaded' class Meta: @@ -219,23 +220,6 @@ def __str__(self): class Meta: verbose_name = _("Topology") verbose_name_plural = _("Topologies") - # triggers = [ - # pgtrigger.Trigger( - # name="update_topology_geom", - # operation=pgtrigger.Update | pgtrigger.Insert, - # when=pgtrigger.After, - # declare=[('stream_geom', 'geometry')], - # func=""" - # SELECT r.geom FROM river_stream r WHERE NEW.stream_id = r.id INTO stream_geom; - # UPDATE description_morphology - # SET geom = ST_LINESUBSTRING(stream_geom, NEW.start_position, NEW.end_position) - # WHERE topology_id = NEW.id; - # UPDATE description_status - # SET geom = ST_LINESUBSTRING(stream_geom, NEW.start_position, NEW.end_position) - # WHERE topology_id = NEW.id; - # RETURN NEW; - # """ - # )] Study.add_property('stations', Station.within_buffer, _("Stations")) diff --git a/georiviere/utils/__init__.py b/georiviere/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/georiviere/utils/mixins/managers.py b/georiviere/utils/mixins/managers.py new file mode 100644 index 00000000..afb01020 --- /dev/null +++ b/georiviere/utils/mixins/managers.py @@ -0,0 +1,7 @@ +from django.db import connection + + +class TruncateManagerMixin: + def truncate(self): + with connection.cursor() as cursor: + cursor.execute('TRUNCATE TABLE "{0}" CASCADE'.format(self.model._meta.db_table))