From a75b440a9c2723d9373a143a40437df42ba1a211 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Tue, 1 Oct 2024 09:49:15 +0700 Subject: [PATCH] Update spw mode to also have new attributes (#161) * Update spw mode to also have new attributes * Fic conflict * Update default fields * Bump version and put the crop file to group folder * Update generate_crop_plan task --- django_project/_version.txt | 2 +- django_project/core/admin.py | 6 +- django_project/core/settings/project.py | 3 + ...antingwindowsignal_last_2_days_and_more.py | 38 ++++++++ django_project/gap/models/crop_insight.py | 87 ++++++++++++++++--- django_project/gap/models/farm_group.py | 4 +- django_project/gap/tasks/crop_insight.py | 3 +- django_project/spw/generator/crop_insight.py | 25 +++++- .../0003_alter_rmodeloutput_type.py | 18 ++++ django_project/spw/models.py | 12 +++ .../spw/tests/test_crop_insight_generator.py | 24 ++++- 11 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py create mode 100644 django_project/spw/migrations/0003_alter_rmodeloutput_type.py diff --git a/django_project/_version.txt b/django_project/_version.txt index 7bcd0e36..6812f812 100644 --- a/django_project/_version.txt +++ b/django_project/_version.txt @@ -1 +1 @@ -0.0.2 \ No newline at end of file +0.0.3 \ No newline at end of file diff --git a/django_project/core/admin.py b/django_project/core/admin.py index 95a9d80f..30af572a 100644 --- a/django_project/core/admin.py +++ b/django_project/core/admin.py @@ -111,8 +111,10 @@ def cancel_background_task(modeladmin, request, queryset): class BackgroundTaskAdmin(admin.ModelAdmin): """Admin class for BackgroundTask model.""" - list_display = ('task_name', 'task_id', 'status', 'started_at', - 'finished_at', 'last_update') + list_display = ( + 'task_name', 'task_id', 'status', 'started_at', + 'finished_at', 'last_update', 'context_id' + ) search_fields = ['task_name', 'status', 'task_id'] actions = [cancel_background_task] list_filter = ["status", "task_name"] diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index 5b146aae..64d26d7b 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -7,6 +7,7 @@ import os # noqa from boto3.s3.transfer import TransferConfig + from .contrib import * # noqa from .utils import absolute_path @@ -69,3 +70,5 @@ STORAGE_DIR_PREFIX = os.environ.get("MINIO_AWS_DIR_PREFIX", "media") if STORAGE_DIR_PREFIX and not STORAGE_DIR_PREFIX.endswith("/"): STORAGE_DIR_PREFIX = f"{STORAGE_DIR_PREFIX}/" + +DATA_UPLOAD_MAX_NUMBER_FIELDS = 1500 diff --git a/django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py b/django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py new file mode 100644 index 00000000..bef01f89 --- /dev/null +++ b/django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.7 on 2024-10-01 02:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('gap', '0024_alter_farm_category_alter_farm_rsvp_status'), + ] + + operations = [ + migrations.AddField( + model_name='farmsuitableplantingwindowsignal', + name='last_2_days', + field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 2 days.', null=True), + ), + migrations.AddField( + model_name='farmsuitableplantingwindowsignal', + name='last_4_days', + field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 4 days.', null=True), + ), + migrations.AddField( + model_name='farmsuitableplantingwindowsignal', + name='today_tomorrow', + field=models.FloatField(blank=True, help_text='The rain accumulationSum for today and tomorrow.', null=True), + ), + migrations.AddField( + model_name='farmsuitableplantingwindowsignal', + name='too_wet_indicator', + field=models.CharField(blank=True, help_text='Too wet indicator.', max_length=512, null=True), + ), + migrations.AlterField( + model_name='farmsuitableplantingwindowsignal', + name='signal', + field=models.CharField(help_text='GoNoGo signals.', max_length=512), + ), + ] diff --git a/django_project/gap/models/crop_insight.py b/django_project/gap/models/crop_insight.py index 79d33edc..13eb69b5 100644 --- a/django_project/gap/models/crop_insight.py +++ b/django_project/gap/models/crop_insight.py @@ -4,7 +4,7 @@ .. note:: Models """ - +import os.path import uuid from datetime import date, timedelta @@ -27,6 +27,14 @@ User = get_user_model() +class FarmGroupIsNotSetException(Exception): + """Farm group is not set.""" + + def __init__(self): # noqa + self.message = 'Farm group is not set.' + super().__init__(self.message) + + def ingestor_file_path(instance, filename): """Return upload path for Ingestor files.""" return f'{settings.STORAGE_DIR_PREFIX}crop-insight/{filename}' @@ -171,7 +179,24 @@ class FarmSuitablePlantingWindowSignal(models.Model): ) signal = models.CharField( max_length=512, - help_text='Signal value of Suitable Planting Window l.' + help_text='GoNoGo signals.' + ) + too_wet_indicator = models.CharField( + max_length=512, + help_text='Too wet indicator.', + null=True, blank=True + ) + last_4_days = models.FloatField( + help_text='The rain accumulationSum for last 4 days.', + null=True, blank=True + ) + last_2_days = models.FloatField( + help_text='The rain accumulationSum for last 2 days.', + null=True, blank=True + ) + today_tomorrow = models.FloatField( + help_text='The rain accumulationSum for today and tomorrow.', + null=True, blank=True ) def __str__(self): @@ -288,6 +313,22 @@ def default_fields(): 'longitude', 'SPWTopMessage', 'SPWDescription', + 'TooWet', + 'last_4_days_mm', + 'last_2_days_mm', + 'today_tomorrow_mm' + ] + + @staticmethod + def default_fields_used(): + """Return list of default fields that being used by crop insight.""" + return [ + 'farmID', + 'phoneNumber', + 'latitude', + 'longitude', + 'SPWTopMessage', + 'SPWDescription' ] def __init__( @@ -362,6 +403,10 @@ def data(self) -> dict: # Spw data spw_top_message = '' spw_description = '' + too_wet = '' + last_4_days = '' + last_2_days = '' + today_tomorrow = '' spw = FarmSuitablePlantingWindowSignal.objects.filter( farm=self.farm, generated_date=self.generated_date @@ -374,15 +419,31 @@ def data(self) -> dict: except SPWOutput.DoesNotExist: spw_top_message = spw.signal spw_description = '' - # --------------------------------------- + if spw.too_wet_indicator is not None: + too_wet = spw.too_wet_indicator + if spw.last_4_days is not None: + last_4_days = spw.last_4_days + if spw.last_2_days is not None: + last_2_days = spw.last_2_days + if spw.today_tomorrow is not None: + today_tomorrow = spw.today_tomorrow + + # --------------------------------------- + # Check default_fields functions + default_fields = CropPlanData.default_fields() output = { - 'farmID': self.farm.unique_id, - 'phoneNumber': self.farm.phone_number, - 'latitude': self.latitude, - 'longitude': self.longitude, - 'SPWTopMessage': spw_top_message, - 'SPWDescription': spw_description, + default_fields[0]: self.farm.unique_id, + default_fields[1]: self.farm.phone_number, + default_fields[2]: self.latitude, + default_fields[3]: self.longitude, + default_fields[4]: spw_top_message, + default_fields[5]: spw_description, + default_fields[6]: too_wet, + default_fields[7]: last_4_days, + default_fields[8]: last_2_days, + default_fields[9]: today_tomorrow + } # ---------------------------------------- @@ -542,9 +603,10 @@ def _generate_report(self): from spw.generator.crop_insight import CropInsightFarmGenerator # If farm is empty, put empty farm - farms = [] if self.farm_group: farms = self.farm_group.farms.all() + else: + raise FarmGroupIsNotSetException() output = [ self.farm_group.headers @@ -583,7 +645,10 @@ def _generate_report(self): for row in output: csv_content += ','.join(map(str, row)) + '\n' content_file = ContentFile(csv_content) - self.file.save(f'{self.unique_id}.csv', content_file) + self.file.save( + os.path.join(f'{self.farm_group.id}', f'{self.unique_id}.csv'), + content_file + ) self.save() # Send email diff --git a/django_project/gap/models/farm_group.py b/django_project/gap/models/farm_group.py index af73e426..fbc68d3d 100644 --- a/django_project/gap/models/farm_group.py +++ b/django_project/gap/models/farm_group.py @@ -51,11 +51,13 @@ def prepare_fields(self): self.farmgroupcropinsightfield_set.all().delete() column_num = 1 for default_field in CropPlanData.default_fields(): + active = default_field in CropPlanData.default_fields_used() FarmGroupCropInsightField.objects.update_or_create( farm_group=self, field=default_field, defaults={ - 'column_number': column_num + 'column_number': column_num, + 'active': active } ) column_num += 1 diff --git a/django_project/gap/tasks/crop_insight.py b/django_project/gap/tasks/crop_insight.py index 46a3788e..d80cff7b 100644 --- a/django_project/gap/tasks/crop_insight.py +++ b/django_project/gap/tasks/crop_insight.py @@ -42,8 +42,7 @@ def generate_crop_plan(): requested_by=user, farm_group=group, ) - # generate report - request.run() + generate_insight_report.delay(request.id) @app.task(name="retry_crop_plan_generators") diff --git a/django_project/spw/generator/crop_insight.py b/django_project/spw/generator/crop_insight.py index e3c6193e..1d8fb214 100644 --- a/django_project/spw/generator/crop_insight.py +++ b/django_project/spw/generator/crop_insight.py @@ -35,14 +35,29 @@ def __init__(self, farm: Farm): self.tomorrow = self.today + timedelta(days=1) self.attributes = calculate_from_point_attrs() - def save_spw(self, signal, farm: Farm): + def return_float(self, value): + """Return float value.""" + try: + return float(value) + except ValueError: + return None + + def save_spw( + self, farm: Farm, signal, too_wet_indicator, last_4_days, + last_2_days, today_tomorrow + ): """Save spw data.""" # Save SPW FarmSuitablePlantingWindowSignal.objects.update_or_create( farm=farm, generated_date=self.today, defaults={ - 'signal': signal + 'signal': signal, + 'too_wet_indicator': too_wet_indicator, + 'last_4_days': self.return_float(last_4_days), + 'last_2_days': self.return_float(last_2_days), + 'today_tomorrow': self.return_float(today_tomorrow) + } ) @@ -124,5 +139,9 @@ def _generate_spw(self): farms = Farm.objects.filter(grid=self.farm.grid) for farm in farms: - self.save_spw(output.data.goNoGo, farm) + self.save_spw( + farm, + output.data.goNoGo, output.data.tooWet, output.data.last4Days, + output.data.last2Days, output.data.todayTomorrow + ) self.save_shortterm_forecast(historical_dict, farm) diff --git a/django_project/spw/migrations/0003_alter_rmodeloutput_type.py b/django_project/spw/migrations/0003_alter_rmodeloutput_type.py new file mode 100644 index 00000000..f4bbd882 --- /dev/null +++ b/django_project/spw/migrations/0003_alter_rmodeloutput_type.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2024-09-27 06:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spw', '0002_spwoutput'), + ] + + operations = [ + migrations.AlterField( + model_name='rmodeloutput', + name='type', + field=models.CharField(choices=[('goNoGo', 'goNoGo'), ('days_h2to_f2', 'days_h2to_f2'), ('days_f3to_f5', 'days_f3to_f5'), ('days_f6to_f13', 'days_f6to_f13'), ('nearDaysLTNPercent', 'nearDaysLTNPercent'), ('nearDaysCurPercent', 'nearDaysCurPercent'), ('tooWet', 'tooWet'), ('last4Days', 'last4Days'), ('last2Days', 'last2Days'), ('todayTomorrow', 'todayTomorrow')], max_length=100), + ), + ] diff --git a/django_project/spw/models.py b/django_project/spw/models.py index 267d8b1b..99a7473c 100644 --- a/django_project/spw/models.py +++ b/django_project/spw/models.py @@ -45,6 +45,10 @@ class RModelOutputType: DAYS_f6TO_F13 = 'days_f6to_f13' NEAR_DAYS_LTN_PERCENT = 'nearDaysLTNPercent' NEAR_DAYS_CUR_PERCENT = 'nearDaysCurPercent' + TOO_WET_STATUS = 'tooWet' + LAST_4_DAYS = 'last4Days' + LAST_2_DAYS = 'last2Days' + TODAY_TOMORROW = 'todayTomorrow' class RModelOutput(models.Model): @@ -66,6 +70,14 @@ class RModelOutput(models.Model): RModelOutputType.NEAR_DAYS_LTN_PERCENT), (RModelOutputType.NEAR_DAYS_CUR_PERCENT, RModelOutputType.NEAR_DAYS_CUR_PERCENT), + (RModelOutputType.TOO_WET_STATUS, + RModelOutputType.TOO_WET_STATUS), + (RModelOutputType.LAST_4_DAYS, + RModelOutputType.LAST_4_DAYS), + (RModelOutputType.LAST_2_DAYS, + RModelOutputType.LAST_2_DAYS), + (RModelOutputType.TODAY_TOMORROW, + RModelOutputType.TODAY_TOMORROW), ) ) variable_name = models.CharField(max_length=100) diff --git a/django_project/spw/tests/test_crop_insight_generator.py b/django_project/spw/tests/test_crop_insight_generator.py index 0599fb47..5e02d099 100644 --- a/django_project/spw/tests/test_crop_insight_generator.py +++ b/django_project/spw/tests/test_crop_insight_generator.py @@ -18,7 +18,8 @@ from gap.factories.farm import FarmFactory, FarmGroupFactory from gap.factories.grid import GridFactory from gap.models.crop_insight import ( - FarmSuitablePlantingWindowSignal, CropInsightRequest + FarmSuitablePlantingWindowSignal, CropInsightRequest, + FarmGroupIsNotSetException ) from gap.models.preferences import Preferences from gap.tasks.crop_insight import ( @@ -183,6 +184,10 @@ def create_timeline_data( 'goNoGo': ['Plant NOW Tier 1b'], 'nearDaysLTNPercent': [10.0], 'nearDaysCurPercent': [60.0], + 'tooWet': ['Likely too wet to plant'], + 'last4Days': [80], + 'last2Days': [60], + 'todayTomorrow': [40], } ) fetch_timelines_data_val = {} @@ -233,6 +238,10 @@ def create_timeline_data( 'goNoGo': ['Do NOT plant, DRY Tier 4b'], 'nearDaysLTNPercent': [10.0], 'nearDaysCurPercent': [60.0], + 'tooWet': ['Too wet to plant'], + 'last4Days': [100], + 'last2Days': [80], + 'todayTomorrow': [80], } ) fetch_timelines_data_val = {} @@ -264,16 +273,29 @@ def create_timeline_data( 'goNoGo': '', 'nearDaysLTNPercent': [10.0], 'nearDaysCurPercent': [60.0], + 'tooWet': '', + 'last4Days': '', + 'last2Days': '', + 'todayTomorrow': '', } ) mock_fetch_timelines_data.return_value = {} + # Farm group is required, raise error + with self.assertRaises(FarmGroupIsNotSetException): + request = CropInsightRequestFactory.create() + generate_insight_report(request.id) + # Crop insight report self.request = CropInsightRequestFactory.create( farm_group=self.farm_group ) generate_insight_report(self.request.id) self.request.refresh_from_db() + + # Check the if of farm group in the path + self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path) + with self.request.file.open(mode='r') as csv_file: csv_reader = csv.reader(csv_file) row_num = 1