Add country to data provider/organisation (#128)

* Add country to data provider/organisation

* Change terms

* Change download file to ods

* Update uploader file to use ods

* Fix link

admin/well_management.py

@@ -24,12 +24,21 @@ def rerun_cache(modeladmin, request, queryset):
         generate_data_organisation_cache(organisation_id=org.id)
 
 
+def reassign_wells_country(modeladmin, request, queryset):
+    for org in queryset.filter(country__isnull=False):
+        org.well_set.all().update(country=org.country)
+
+
 class OrganisationAdmin(admin.ModelAdmin):
-    list_display = ('name', 'active')
+    list_display = ('name', 'active', 'country', 'well_number')
     list_editable = ('active',)
-    actions = (rerun_cache,)
+    list_filter = ('country',)
+    actions = (rerun_cache, reassign_wells_country)
     form = OrganisationFormAdmin
 
+    def well_number(self, org: Organisation):
+        return org.well_set.all().count()
+
 
 def fetch(modeladmin, request, queryset):
     for user in User.objects.exclude(

forms/organisation.py

@@ -23,7 +23,9 @@ class OrganisationFormAdmin(forms.ModelForm):
 
     class Meta:
         model = Organisation
-        fields = ('name', 'description', 'admin_users', 'editor_users')
+        fields = (
+            'name', 'country', 'description', 'admin_users', 'editor_users'
+        )
 
     def __init__(self, *args, **kwargs):
         forms.ModelForm.__init__(self, *args, **kwargs)

forms/uploader/csv_well_form.py

@@ -17,19 +17,25 @@ class CsvWellForm(forms.Form):
 
     gw_well_file = forms.FileField(
         label=_("General information"),
-        widget=forms.FileInput(attrs={'class': 'form-control'}),
-        required=False
+        widget=forms.FileInput(
+            attrs={'class': 'form-control', 'accept': '.ods'}
+        ),
+        required=False,
     )
 
     gw_well_monitoring_file = forms.FileField(
         label=_("Monitoring data"),
-        widget=forms.FileInput(attrs={'class': 'form-control'}),
+        widget=forms.FileInput(
+            attrs={'class': 'form-control', 'accept': '.ods'}
+        ),
         required=False
     )
 
     gw_well_drilling_and_construction_file = forms.FileField(
         label=_("Drilling and donstruction data"),
-        widget=forms.FileInput(attrs={'class': 'form-control'}),
+        widget=forms.FileInput(
+            attrs={'class': 'form-control', 'accept': '.ods'}
+        ),
         required=False
     )
 

migrations/0083_organisation_country.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.20 on 2024-04-17 08:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gwml2', '0082_views'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organisation',
+            name='country',
+            field=models.ForeignKey(blank=True, help_text="Identify the country of the organisation. It is being used to assign well's country under this organisation. If this is empty, all well under this organisation will be assigned based on geometry of country.", null=True, on_delete=django.db.models.deletion.SET_NULL, to='gwml2.country'),
+        ),
+    ]

models/upload_session.py

@@ -18,6 +18,7 @@
 from gwml2.models.metadata.license_metadata import LicenseMetadata
 from gwml2.models.well import Well
 from gwml2.models.well_management.organisation import Organisation
+from gwml2.utilities import ods_to_xlsx, xlsx_to_ods
 
 UPLOAD_SESSION_CATEGORY_WELL_UPLOAD = 'well_upload'
 UPLOAD_SESSION_CATEGORY_MONITORING_UPLOAD = 'well_monitoring_upload'
@@ -283,7 +284,7 @@ def run(self, restart: bool = False):
     def create_report_excel(self):
         """Created excel that will contain reports."""
 
-        _file = self.upload_file.path
+        _file = ods_to_xlsx(self.upload_file.path)
         _report_file = _file.replace('.xls', '.report.xls')
         if os.path.exists(_report_file):
             os.remove(_report_file)
@@ -308,6 +309,11 @@ def create_report_excel(self):
                 row_status.update_sheet(worksheet, status_column_idx)
         workbook.save(_report_file)
         os.chmod(_report_file, 0o0777)
+        xlsx_to_ods(_report_file)
+
+        # Delete files
+        os.remove(_file)
+        os.remove(_report_file)
 
 
 RowStatus = [

models/well.py

@@ -125,7 +125,15 @@ class Meta:
     def assign_country(self, force=False):
         """Assign country to the well."""
         from gwml2.models.general_information import Country
-        if not self.country or force:
+        # Autoassign with organisation country
+        if (
+                self.organisation and self.organisation.country and
+                self.organisation.country != self.country
+        ):
+            self.country = self.organisation.country
+            self.save()
+        # Check country from the geometry
+        elif not self.country or force:
             country = Country.objects.filter(
                 geometry__contains=self.location
             ).first()

models/well_management/organisation.py

@@ -1,6 +1,8 @@
 from django.contrib.gis.db import models
 from django.contrib.postgres.fields import ArrayField
 
+from gwml2.models.general import Country
+
 
 class Organisation(models.Model):
     """ Organisation
@@ -25,6 +27,17 @@ class Organisation(models.Model):
     editors = ArrayField(
         models.IntegerField(), default=list, null=True)
 
+    country = models.ForeignKey(
+        Country, null=True, blank=True, on_delete=models.SET_NULL,
+        help_text=(
+            'Identify the country of the organisation. '
+            "It is being used to assign well's country under "
+            "this organisation. "
+            'If this is empty, all well under this organisation '
+            'will be assigned based on geometry of country.'
+        )
+    )
+
     class Meta:
         db_table = 'organisation'
         ordering = ['name']

tasks/data_file_cache/base_cache.py

@@ -11,6 +11,8 @@
 from gwml2.models.download_request import WELL_AND_MONITORING_DATA, GGMN
 from gwml2.models.term import TermFeatureType
 from gwml2.models.well import Organisation
+from gwml2.terms import SheetName
+from gwml2.utilities import xlsx_to_ods
 
 GWML2_FOLDER = settings.GWML2_FOLDER
 WELL_FOLDER = os.path.join(GWML2_FOLDER, 'wells-data')
@@ -52,7 +54,7 @@ class WellCacheFileBase(object):
     current_time = None
     wells_filename = 'wells.xlsx'
     drill_filename = 'drilling_and_construction.xlsx'
-    monitor_filename = 'monitoring_data.xlsx'
+    monitor_filename = 'monitoring_data.ods'
 
     # cache
     feature_types = {}
@@ -178,6 +180,18 @@ def clear_folder(self):
             except FileNotFoundError:
                 pass
 
+    def zip_excel_to_ods(self, zip_file, filename, data_type):
+        """Zip excel to ods."""
+        well_file = self.file_by_type(
+            filename, data_type
+        )
+        xlsx_to_ods(well_file)
+        zip_file.write(
+            well_file.replace('.xlsx', '.ods'),
+            filename.replace('.xlsx', '.ods'),
+            compress_type=zipfile.ZIP_DEFLATED
+        )
+
     def run(self):
         self.current_time = time.time()
         self.log(
@@ -234,8 +248,9 @@ def run(self):
                     ggmn_organisations_list
                 ) and well.organisation else None,
                 [
-                    'Drilling and Construction', 'Water Strike',
-                    'Stratigraphic Log', 'Structures'
+                    SheetName.drilling_and_construction,
+                    'Water Strike', 'Stratigraphic Log',
+                    SheetName.structure
                 ]
             )
 
@@ -270,21 +285,11 @@ def run(self):
                 if not zip_file:
                     zip_file = zipfile.ZipFile(zip_filepath, 'w')
 
-                    well_file = self.file_by_type(
-                        self.wells_filename, data_type
+                    self.zip_excel_to_ods(
+                        zip_file, self.wells_filename, data_type
                     )
-                    zip_file.write(
-                        well_file, self.wells_filename,
-                        compress_type=zipfile.ZIP_DEFLATED
-                    )
-
-                    drill_file = self.file_by_type(
-                        self.drill_filename, data_type
-                    )
-                    zip_file.write(
-                        drill_file,
-                        self.drill_filename,
-                        compress_type=zipfile.ZIP_DEFLATED
+                    self.zip_excel_to_ods(
+                        zip_file, self.drill_filename, data_type
                     )
 
                 if well.number_of_measurements == 0:
@@ -299,11 +304,11 @@ def run(self):
                     try:
                         _filename = (
                             f'monitoring/{original_id} '
-                            f'({original_ids_found[original_id] + 1}).xlsx'
+                            f'({original_ids_found[original_id] + 1}).ods'
                         )
                         continue
                     except KeyError:
-                        _filename = f'monitoring/{original_id}.xlsx'
+                        _filename = f'monitoring/{original_id}.ods'
                         original_ids_found[original_id] = 0
 
                     zip_file.write(

tasks/data_file_cache/wells_cache.py

@@ -26,6 +26,8 @@
 from gwml2.tasks.data_file_cache.organisation_cache import (
     generate_data_organisation_cache
 )
+from gwml2.terms import SheetName
+from gwml2.utilities import xlsx_to_ods
 
 GWML2_FOLDER = settings.GWML2_FOLDER
 WELL_FOLDER = os.path.join(GWML2_FOLDER, 'wells-data')
@@ -130,6 +132,9 @@ def run(self):
         monitor_book.active = 0
         monitor_book.save(monitor_file)
 
+        xlsx_to_ods(monitor_file)
+        os.remove(monitor_file)
+
     def write_json(self, folder, sheetname, data):
         """Write json by sheetname."""
         _file = os.path.join(folder, sheetname + '.json')
@@ -311,8 +316,8 @@ def drilling_and_construction(self, folder, well):
             construction.pump_installer if construction else '',
             construction.pump_description if construction else '',
         ]
-        sheetname = 'Drilling and Construction'
-        self.write_json(folder, sheetname, [data])
+
+        self.write_json(folder, SheetName.drilling_and_construction, [data])
 
         # --------------------------------------------------------------------------
         # For drilling data
@@ -371,7 +376,7 @@ def drilling_and_construction(self, folder, well):
         # --------------------------------------------------------------------------
         # For Construction Data
         if construction:
-            sheetname = 'Structures'
+            sheetname = SheetName.structure
             sheet_data = []
             for structure in well.construction.constructionstructure_set.all():
                 top_depth = structure.top_depth

tasks/uploader/base.py

@@ -1,6 +1,7 @@
 import json
 
 from celery.utils.log import get_task_logger
+from django.db.models import Q
 
 from gwml2.models.general import Unit, Country
 from gwml2.models.term import (
@@ -10,9 +11,7 @@
 from gwml2.models.upload_session import (
     UploadSession, UploadSessionRowStatus
 )
-from gwml2.models.well import (
-    Well
-)
+from gwml2.models.well import Well
 from gwml2.tasks.uploader.well import get_column
 from gwml2.views.form_group.form_group import FormNotValid
 from gwml2.views.groundwater_form import WellEditing
@@ -251,6 +250,10 @@ def _convert_record(self, sheet_name, record):
                             rel_value = TERM.objects.get(
                                 name__iexact=value
                             ).name
+                        elif TERM == Country:
+                            rel_value = TERM.objects.get(
+                                Q(name__iexact=value) | Q(code__iexact=value)
+                            ).id
                         else:
                             rel_value = TERM.objects.get(name__iexact=value).id
                         cache[value] = rel_value

tasks/uploader/drilling_and_construction.py

@@ -3,14 +3,15 @@
 from gwml2.models.general import Unit
 from gwml2.models.term import TermDrillingMethod
 from gwml2.tasks.uploader.base import BaseUploader
+from gwml2.terms import SheetName
 
 logger = get_task_logger(__name__)
 
 
 class DrillingAndConstructionUploader(BaseUploader):
     """ Save well uploader from excel """
     UPLOADER_NAME = 'Drilling and construction'
-    SHEETS = ['Drilling and Construction']
+    SHEETS = [SheetName.drilling_and_construction]
 
     # key related with the index of keys
     # value if it has tem

tasks/uploader/structures.py

@@ -5,6 +5,7 @@
     TermReferenceElevationType, TermConstructionStructureType
 )
 from gwml2.tasks.uploader.base import BaseUploader
+from gwml2.terms import SheetName
 
 logger = get_task_logger(__name__)
 
@@ -13,7 +14,7 @@ class StructuresUploader(BaseUploader):
     """ Save well uploader from excel """
     UPLOADER_NAME = 'Drilling and construction'
     IS_OPTIONAL = True
-    SHEETS = ['Structures']
+    SHEETS = [SheetName.structure]
 
     # key related with the index of keys
     # value if it has tem

tasks/uploader/uploader.py

@@ -1,7 +1,6 @@
 from celery.utils.log import get_task_logger
 from django.db.models.signals import post_save
-from pyexcel_xls import get_data as xls_get
-from pyexcel_xlsx import get_data as xlsx_get
+from pyexcel_ods3 import get_data
 
 from gwml2.models.upload_session import (
     UploadSession, UploadSessionCancelled
@@ -82,11 +81,7 @@ def process(
         _file = self.upload_session.upload_file
         if _file:
             _file.seek(0)
-            records = None
-            if str(_file).split('.')[-1] == 'xls':
-                records = xls_get(_file, column_limit=20)
-            elif str(_file).split('.')[-1] == 'xlsx':
-                records = xlsx_get(_file, column_limit=20)
+            records = get_data(_file.path)
             if not records:
                 error = 'No data found.'
                 self.upload_session.update_progress(

templates/groundwater_form/drilling_and_construction.html

@@ -27,7 +27,7 @@ <h3>
         {% include "groundwater_form/well_chart.html" %}
         {% include "groundwater_form/_many-to-many.html" with id='water_strike' title=_('Water Strike') theform=water_strike help_text='' delete='yes' %}
         {% include "groundwater_form/_many-to-many.html" with id='stratigraphic_log' title=_('Stratigraphic Log') theform=stratigraphic_log help_text='' delete='yes' %}
-        {% include "groundwater_form/_many-to-many.html" with id='structure' title=_('Design') theform=structure thedata=structures help_text='' delete='yes' %}
+        {% include "groundwater_form/_many-to-many.html" with id='structure' title=_('Construction') theform=structure thedata=structures help_text='' delete='yes' %}
     </div>
     <div id="construction">
         <div id="construction-data" class="section">