Update spw mode to also have new attributes (#161)

* Update spw mode to also have new attributes

* Fic conflict

* Update default fields

* Bump version and put the crop file to group folder

* Update generate_crop_plan task

django_project/_version.txt

@@ -1 +1 @@
-0.0.2
+0.0.3

django_project/core/admin.py

@@ -111,8 +111,10 @@ def cancel_background_task(modeladmin, request, queryset):
 class BackgroundTaskAdmin(admin.ModelAdmin):
     """Admin class for BackgroundTask model."""
 
-    list_display = ('task_name', 'task_id', 'status', 'started_at',
-                    'finished_at', 'last_update')
+    list_display = (
+        'task_name', 'task_id', 'status', 'started_at',
+        'finished_at', 'last_update', 'context_id'
+    )
     search_fields = ['task_name', 'status', 'task_id']
     actions = [cancel_background_task]
     list_filter = ["status", "task_name"]

django_project/core/settings/project.py

@@ -7,6 +7,7 @@
 import os  # noqa
 
 from boto3.s3.transfer import TransferConfig
+
 from .contrib import *  # noqa
 from .utils import absolute_path
 
@@ -69,3 +70,5 @@
 STORAGE_DIR_PREFIX = os.environ.get("MINIO_AWS_DIR_PREFIX", "media")
 if STORAGE_DIR_PREFIX and not STORAGE_DIR_PREFIX.endswith("/"):
     STORAGE_DIR_PREFIX = f"{STORAGE_DIR_PREFIX}/"
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 1500

django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.7 on 2024-10-01 02:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gap', '0024_alter_farm_category_alter_farm_rsvp_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='farmsuitableplantingwindowsignal',
+            name='last_2_days',
+            field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 2 days.', null=True),
+        ),
+        migrations.AddField(
+            model_name='farmsuitableplantingwindowsignal',
+            name='last_4_days',
+            field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 4 days.', null=True),
+        ),
+        migrations.AddField(
+            model_name='farmsuitableplantingwindowsignal',
+            name='today_tomorrow',
+            field=models.FloatField(blank=True, help_text='The rain accumulationSum for today and tomorrow.', null=True),
+        ),
+        migrations.AddField(
+            model_name='farmsuitableplantingwindowsignal',
+            name='too_wet_indicator',
+            field=models.CharField(blank=True, help_text='Too wet indicator.', max_length=512, null=True),
+        ),
+        migrations.AlterField(
+            model_name='farmsuitableplantingwindowsignal',
+            name='signal',
+            field=models.CharField(help_text='GoNoGo signals.', max_length=512),
+        ),
+    ]

django_project/gap/models/crop_insight.py

@@ -4,7 +4,7 @@
 
 .. note:: Models
 """
-
+import os.path
 import uuid
 from datetime import date, timedelta
 
@@ -27,6 +27,14 @@
 User = get_user_model()
 
 
+class FarmGroupIsNotSetException(Exception):
+    """Farm group is not set."""
+
+    def __init__(self):  # noqa
+        self.message = 'Farm group is not set.'
+        super().__init__(self.message)
+
+
 def ingestor_file_path(instance, filename):
     """Return upload path for Ingestor files."""
     return f'{settings.STORAGE_DIR_PREFIX}crop-insight/{filename}'
@@ -171,7 +179,24 @@ class FarmSuitablePlantingWindowSignal(models.Model):
     )
     signal = models.CharField(
         max_length=512,
-        help_text='Signal value of Suitable Planting Window l.'
+        help_text='GoNoGo signals.'
+    )
+    too_wet_indicator = models.CharField(
+        max_length=512,
+        help_text='Too wet indicator.',
+        null=True, blank=True
+    )
+    last_4_days = models.FloatField(
+        help_text='The rain accumulationSum for last 4 days.',
+        null=True, blank=True
+    )
+    last_2_days = models.FloatField(
+        help_text='The rain accumulationSum for last 2 days.',
+        null=True, blank=True
+    )
+    today_tomorrow = models.FloatField(
+        help_text='The rain accumulationSum for today and tomorrow.',
+        null=True, blank=True
     )
 
     def __str__(self):
@@ -288,6 +313,22 @@ def default_fields():
             'longitude',
             'SPWTopMessage',
             'SPWDescription',
+            'TooWet',
+            'last_4_days_mm',
+            'last_2_days_mm',
+            'today_tomorrow_mm'
+        ]
+
+    @staticmethod
+    def default_fields_used():
+        """Return list of default fields that being used by crop insight."""
+        return [
+            'farmID',
+            'phoneNumber',
+            'latitude',
+            'longitude',
+            'SPWTopMessage',
+            'SPWDescription'
         ]
 
     def __init__(
@@ -362,6 +403,10 @@ def data(self) -> dict:
         # Spw data
         spw_top_message = ''
         spw_description = ''
+        too_wet = ''
+        last_4_days = ''
+        last_2_days = ''
+        today_tomorrow = ''
         spw = FarmSuitablePlantingWindowSignal.objects.filter(
             farm=self.farm,
             generated_date=self.generated_date
@@ -374,15 +419,31 @@ def data(self) -> dict:
             except SPWOutput.DoesNotExist:
                 spw_top_message = spw.signal
                 spw_description = ''
-        # ---------------------------------------
 
+            if spw.too_wet_indicator is not None:
+                too_wet = spw.too_wet_indicator
+            if spw.last_4_days is not None:
+                last_4_days = spw.last_4_days
+            if spw.last_2_days is not None:
+                last_2_days = spw.last_2_days
+            if spw.today_tomorrow is not None:
+                today_tomorrow = spw.today_tomorrow
+
+        # ---------------------------------------
+        # Check default_fields functions
+        default_fields = CropPlanData.default_fields()
         output = {
-            'farmID': self.farm.unique_id,
-            'phoneNumber': self.farm.phone_number,
-            'latitude': self.latitude,
-            'longitude': self.longitude,
-            'SPWTopMessage': spw_top_message,
-            'SPWDescription': spw_description,
+            default_fields[0]: self.farm.unique_id,
+            default_fields[1]: self.farm.phone_number,
+            default_fields[2]: self.latitude,
+            default_fields[3]: self.longitude,
+            default_fields[4]: spw_top_message,
+            default_fields[5]: spw_description,
+            default_fields[6]: too_wet,
+            default_fields[7]: last_4_days,
+            default_fields[8]: last_2_days,
+            default_fields[9]: today_tomorrow
+
         }
 
         # ----------------------------------------
@@ -542,9 +603,10 @@ def _generate_report(self):
         from spw.generator.crop_insight import CropInsightFarmGenerator
 
         # If farm is empty, put empty farm
-        farms = []
         if self.farm_group:
             farms = self.farm_group.farms.all()
+        else:
+            raise FarmGroupIsNotSetException()
 
         output = [
             self.farm_group.headers
@@ -583,7 +645,10 @@ def _generate_report(self):
         for row in output:
             csv_content += ','.join(map(str, row)) + '\n'
         content_file = ContentFile(csv_content)
-        self.file.save(f'{self.unique_id}.csv', content_file)
+        self.file.save(
+            os.path.join(f'{self.farm_group.id}', f'{self.unique_id}.csv'),
+            content_file
+        )
         self.save()
 
         # Send email

django_project/gap/models/farm_group.py

@@ -51,11 +51,13 @@ def prepare_fields(self):
         self.farmgroupcropinsightfield_set.all().delete()
         column_num = 1
         for default_field in CropPlanData.default_fields():
+            active = default_field in CropPlanData.default_fields_used()
             FarmGroupCropInsightField.objects.update_or_create(
                 farm_group=self,
                 field=default_field,
                 defaults={
-                    'column_number': column_num
+                    'column_number': column_num,
+                    'active': active
                 }
             )
             column_num += 1

django_project/gap/tasks/crop_insight.py

@@ -42,8 +42,7 @@ def generate_crop_plan():
             requested_by=user,
             farm_group=group,
         )
-        # generate report
-        request.run()
+        generate_insight_report.delay(request.id)
 
 
 @app.task(name="retry_crop_plan_generators")

django_project/spw/generator/crop_insight.py

@@ -35,14 +35,29 @@ def __init__(self, farm: Farm):
         self.tomorrow = self.today + timedelta(days=1)
         self.attributes = calculate_from_point_attrs()
 
-    def save_spw(self, signal, farm: Farm):
+    def return_float(self, value):
+        """Return float value."""
+        try:
+            return float(value)
+        except ValueError:
+            return None
+
+    def save_spw(
+            self, farm: Farm, signal, too_wet_indicator, last_4_days,
+            last_2_days, today_tomorrow
+    ):
         """Save spw data."""
         # Save SPW
         FarmSuitablePlantingWindowSignal.objects.update_or_create(
             farm=farm,
             generated_date=self.today,
             defaults={
-                'signal': signal
+                'signal': signal,
+                'too_wet_indicator': too_wet_indicator,
+                'last_4_days': self.return_float(last_4_days),
+                'last_2_days': self.return_float(last_2_days),
+                'today_tomorrow': self.return_float(today_tomorrow)
+
             }
         )
 
@@ -124,5 +139,9 @@ def _generate_spw(self):
             farms = Farm.objects.filter(grid=self.farm.grid)
 
         for farm in farms:
-            self.save_spw(output.data.goNoGo, farm)
+            self.save_spw(
+                farm,
+                output.data.goNoGo, output.data.tooWet, output.data.last4Days,
+                output.data.last2Days, output.data.todayTomorrow
+            )
             self.save_shortterm_forecast(historical_dict, farm)

django_project/spw/migrations/0003_alter_rmodeloutput_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-09-27 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('spw', '0002_spwoutput'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='rmodeloutput',
+            name='type',
+            field=models.CharField(choices=[('goNoGo', 'goNoGo'), ('days_h2to_f2', 'days_h2to_f2'), ('days_f3to_f5', 'days_f3to_f5'), ('days_f6to_f13', 'days_f6to_f13'), ('nearDaysLTNPercent', 'nearDaysLTNPercent'), ('nearDaysCurPercent', 'nearDaysCurPercent'), ('tooWet', 'tooWet'), ('last4Days', 'last4Days'), ('last2Days', 'last2Days'), ('todayTomorrow', 'todayTomorrow')], max_length=100),
+        ),
+    ]

django_project/spw/models.py

@@ -45,6 +45,10 @@ class RModelOutputType:
     DAYS_f6TO_F13 = 'days_f6to_f13'
     NEAR_DAYS_LTN_PERCENT = 'nearDaysLTNPercent'
     NEAR_DAYS_CUR_PERCENT = 'nearDaysCurPercent'
+    TOO_WET_STATUS = 'tooWet'
+    LAST_4_DAYS = 'last4Days'
+    LAST_2_DAYS = 'last2Days'
+    TODAY_TOMORROW = 'todayTomorrow'
 
 
 class RModelOutput(models.Model):
@@ -66,6 +70,14 @@ class RModelOutput(models.Model):
              RModelOutputType.NEAR_DAYS_LTN_PERCENT),
             (RModelOutputType.NEAR_DAYS_CUR_PERCENT,
              RModelOutputType.NEAR_DAYS_CUR_PERCENT),
+            (RModelOutputType.TOO_WET_STATUS,
+             RModelOutputType.TOO_WET_STATUS),
+            (RModelOutputType.LAST_4_DAYS,
+             RModelOutputType.LAST_4_DAYS),
+            (RModelOutputType.LAST_2_DAYS,
+             RModelOutputType.LAST_2_DAYS),
+            (RModelOutputType.TODAY_TOMORROW,
+             RModelOutputType.TODAY_TOMORROW),
         )
     )
     variable_name = models.CharField(max_length=100)

django_project/spw/tests/test_crop_insight_generator.py

@@ -18,7 +18,8 @@
 from gap.factories.farm import FarmFactory, FarmGroupFactory
 from gap.factories.grid import GridFactory
 from gap.models.crop_insight import (
-    FarmSuitablePlantingWindowSignal, CropInsightRequest
+    FarmSuitablePlantingWindowSignal, CropInsightRequest,
+    FarmGroupIsNotSetException
 )
 from gap.models.preferences import Preferences
 from gap.tasks.crop_insight import (
@@ -183,6 +184,10 @@ def create_timeline_data(
                 'goNoGo': ['Plant NOW Tier 1b'],
                 'nearDaysLTNPercent': [10.0],
                 'nearDaysCurPercent': [60.0],
+                'tooWet': ['Likely too wet to plant'],
+                'last4Days': [80],
+                'last2Days': [60],
+                'todayTomorrow': [40],
             }
         )
         fetch_timelines_data_val = {}
@@ -233,6 +238,10 @@ def create_timeline_data(
                 'goNoGo': ['Do NOT plant, DRY Tier 4b'],
                 'nearDaysLTNPercent': [10.0],
                 'nearDaysCurPercent': [60.0],
+                'tooWet': ['Too wet to plant'],
+                'last4Days': [100],
+                'last2Days': [80],
+                'todayTomorrow': [80],
             }
         )
         fetch_timelines_data_val = {}
@@ -264,16 +273,29 @@ def create_timeline_data(
                 'goNoGo': '',
                 'nearDaysLTNPercent': [10.0],
                 'nearDaysCurPercent': [60.0],
+                'tooWet': '',
+                'last4Days': '',
+                'last2Days': '',
+                'todayTomorrow': '',
             }
         )
         mock_fetch_timelines_data.return_value = {}
 
+        # Farm group is required, raise error
+        with self.assertRaises(FarmGroupIsNotSetException):
+            request = CropInsightRequestFactory.create()
+            generate_insight_report(request.id)
+
         # Crop insight report
         self.request = CropInsightRequestFactory.create(
             farm_group=self.farm_group
         )
         generate_insight_report(self.request.id)
         self.request.refresh_from_db()
+
+        # Check the if of farm group in the path
+        self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path)
+
         with self.request.file.open(mode='r') as csv_file:
             csv_reader = csv.reader(csv_file)
             row_num = 1