Ajout d'une colonne activity_datetime_utc dans la table logbook reports (#3548)

## Linked issues

- Resolve #3209

Author: VincentAntoine

backend/src/main/resources/db/migration/internal/V0.273__Update_logbook_reports_table.sql

@@ -0,0 +1,23 @@
+ALTER TABLE public.logbook_reports
+  ADD COLUMN activity_datetime_utc TIMESTAMP WITHOUT TIME ZONE;
+
+-- Requête à exécuter après MEP pour ne pas créer de downtime pendant la MEP
+
+-- UPDATE logbook_reports
+-- SET activity_datetime_utc = CASE
+--         WHEN log_type = 'DEP' THEN (value->>'departureDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'NOT-COE' THEN (value->>'effortZoneEntryDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'COE' THEN (value->>'effortZoneEntryDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'FAR' THEN (SELECT MIN((haul->>'farDatetimeUtc')::TIMESTAMPTZ) AT TIME ZONE 'UTC' FROM jsonb_array_elements(value->'hauls') haul)
+--         WHEN log_type = 'CPS' THEN (value->>'cpsDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'DIS' THEN (value->>'discardDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'NOT-COX' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'COX' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'CRO' THEN (value->>'effortZoneExitDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'EOF' THEN (value->>'endOfFishingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'PNO' THEN (value->>'predictedArrivalDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'LAN' THEN (value->>'landingDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         WHEN log_type = 'RTP' THEN (value->>'returnDatetimeUtc')::TIMESTAMPTZ AT TIME ZONE 'UTC'
+--         ELSE NULL
+--     END
+-- WHERE log_type IS NOT NULL

datascience/src/pipeline/parsers/ers/log_parsers.py

@@ -9,7 +9,8 @@
 )
 from src.pipeline.parsers.utils import (
     get_root_tag,
-    make_datetime_json_serializable,
+    make_datetime,
+    serialize_datetime,
     tagged_children,
     try_float,
 )
@@ -23,7 +24,8 @@ def parse_dep(dep):
     date = dep.get("DA")
     time = dep.get("TI")
     # cannot use DateTime because the data needs to be json serializable
-    departure_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    departure_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {
         "departureDatetimeUtc": departure_datetime_utc,
@@ -41,15 +43,20 @@ def parse_dep(dep):
         species_onboard = [parse_spe(spe) for spe in children["SPE"]]
         value["speciesOnboard"] = species_onboard
 
-    data = {"log_type": "DEP", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "DEP",
+        "value": value,
+    }
 
     return data
 
 
 def parse_far(far):
     date = far.get("DA")
     time = far.get("TI")
-    far_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    far_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {"farDatetimeUtc": far_datetime_utc}
 
@@ -72,15 +79,20 @@ def parse_far(far):
         value["latitude"] = try_float(lat)
         value["longitude"] = try_float(lon)
 
-    data = {"log_type": "FAR", "value": {"hauls": [value]}}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "FAR",
+        "value": {"hauls": [value]},
+    }
 
     return data
 
 
 def parse_ecps(ecps):
     date = ecps.get("DA")
     time = ecps.get("TI")
-    far_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    far_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {"cpsDatetimeUtc": far_datetime_utc}
 
@@ -103,15 +115,20 @@ def parse_ecps(ecps):
         value["latitude"] = try_float(lat)
         value["longitude"] = try_float(lon)
 
-    data = {"log_type": "CPS", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "CPS",
+        "value": value,
+    }
 
     return data
 
 
 def parse_dis(dis):
     date = dis.get("DA")
     time = dis.get("TI")
-    discard_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    discard_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {"discardDatetimeUtc": discard_datetime_utc}
 
@@ -121,15 +138,20 @@ def parse_dis(dis):
         catches = [parse_spe(spe) for spe in children["SPE"]]
         value["catches"] = catches
 
-    data = {"log_type": "DIS", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "DIS",
+        "value": value,
+    }
 
     return data
 
 
 def parse_coe(coe):
     date = coe.get("DA")
     time = coe.get("TI")
-    effort_zone_entry_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    effort_zone_entry_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     children = tagged_children(coe)
 
@@ -154,15 +176,20 @@ def parse_coe(coe):
         value["latitudeEntered"] = try_float(lat)
         value["longitudeEntered"] = try_float(lon)
 
-    data = {"log_type": "COE", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "COE",
+        "value": value,
+    }
 
     return data
 
 
 def parse_cox(cox):
     date = cox.get("DA")
     time = cox.get("TI")
-    effort_zone_exit_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    effort_zone_exit_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     children = tagged_children(cox)
 
@@ -187,7 +214,11 @@ def parse_cox(cox):
         value["latitudeExited"] = try_float(lat)
         value["longitudeExited"] = try_float(lon)
 
-    data = {"log_type": "COX", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "COX",
+        "value": value,
+    }
 
     return data
 
@@ -197,6 +228,8 @@ def parse_cro(cro):
 
     value = {}
 
+    activity_datetime_utc = None
+
     if "COE" in children:
         assert len(children["COE"]) == 1
         coe = children["COE"][0]
@@ -207,24 +240,31 @@ def parse_cro(cro):
         assert len(children["COX"]) == 1
         cox = children["COX"][0]
         cox_data = parse_cox(cox)
+        activity_datetime_utc = cox_data["activity_datetime_utc"]
         cox_value = cox_data["value"]
         value = {**value, **cox_value}
 
-    data = {"log_type": "CRO", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "CRO",
+        "value": value,
+    }
     return data
 
 
 def parse_pno(pno):
     date = pno.get("PD")
     time = pno.get("PT")
-    predicted_arrival_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    predicted_arrival_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     date = pno.get("DA")
     time = pno.get("TI")
-    predicted_landing_datetime_utc = make_datetime_json_serializable(date, time)
+    predicted_landing_datetime_utc = serialize_datetime(make_datetime(date, time))
 
     start_date = pno.get("DS")
-    trip_start_date = make_datetime_json_serializable(start_date, None)
+    activity_datetime_utc = make_datetime(start_date, None)
+    trip_start_date = serialize_datetime(activity_datetime_utc)
 
     children = tagged_children(pno)
 
@@ -257,15 +297,20 @@ def parse_pno(pno):
         value["latitude"] = try_float(lat)
         value["longitude"] = try_float(lon)
 
-    data = {"log_type": "PNO", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "PNO",
+        "value": value,
+    }
 
     return data
 
 
 def parse_lan(lan):
     date = lan.get("DA")
     time = lan.get("TI")
-    landing_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    landing_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {
         "landingDatetimeUtc": landing_datetime_utc,
@@ -279,24 +324,34 @@ def parse_lan(lan):
         catches = [parse_spe(spe) for spe in children["SPE"]]
         value["catchLanded"] = catches
 
-    data = {"log_type": "LAN", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "LAN",
+        "value": value,
+    }
 
     return data
 
 
 def parse_eof(eof):
     date = eof.get("DA")
     time = eof.get("TI")
-    end_of_fishing_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    end_of_fishing_datetime_utc = serialize_datetime(activity_datetime_utc)
     value = {"endOfFishingDatetimeUtc": end_of_fishing_datetime_utc}
-    data = {"log_type": "EOF", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "EOF",
+        "value": value,
+    }
     return data
 
 
 def parse_rtp(rtp):
     date = rtp.get("DA")
     time = rtp.get("TI")
-    return_datetime_utc = make_datetime_json_serializable(date, time)
+    activity_datetime_utc = make_datetime(date, time)
+    return_datetime_utc = serialize_datetime(activity_datetime_utc)
 
     value = {
         "returnDatetimeUtc": return_datetime_utc,
@@ -310,6 +365,10 @@ def parse_rtp(rtp):
         gear = [parse_gea(gea) for gea in children["GEA"]]
         value["gearOnboard"] = gear
 
-    data = {"log_type": "RTP", "value": value}
+    data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        "log_type": "RTP",
+        "value": value,
+    }
 
     return data

datascience/src/pipeline/parsers/flux/flux.py

@@ -9,6 +9,7 @@
 from xml.etree.ElementTree import ParseError
 
 import pandas as pd
+from dateutil.parser import parse
 
 from src.pipeline.parsers.flux.log_parsers import (
     null_parser,
@@ -68,7 +69,6 @@ def get_fishing_activity_type(fishing_activity: ET.Element) -> FluxFishingActivi
 
 
 def get_fa_report_type(fa_report_document: ET.Element) -> FluxFAReportDocumentType:
-
     report_type = get_text(
         fa_report_document, './/ram:TypeCode[@listID="FLUX_FA_REPORT_TYPE"]'
     )
@@ -169,7 +169,6 @@ def get_operation_type(xml_element):
 
 
 def parse_metadata(fa_report_document: xml.etree.ElementTree.Element):
-
     metadata = {
         "operation_type": get_operation_type(fa_report_document),
         "report_id": get_text(fa_report_document, './/ram:ID[@schemeID="UUID"]'),
@@ -208,15 +207,18 @@ def parse_fa_report_document(fa_report_document: ET.Element):
 
     children = tagged_children(fa_report_document)
 
+    activity_datetimes_utc = []
     if "SpecifiedFishingActivity" in children:
         log_types = set()
         values = []
         for specified_fishing_activity in children["SpecifiedFishingActivity"]:
-            log_type, value = parse_specified_fishing_activity(
+            log_type, activity_datetime_utc, value = parse_specified_fishing_activity(
                 specified_fishing_activity, report_type
             )
             log_types.add(log_type)
             values.append(value)
+            if isinstance(activity_datetime_utc, datetime):
+                activity_datetimes_utc.append(activity_datetime_utc)
         try:
             assert len(log_types) == 1
         except AssertionError:
@@ -235,7 +237,16 @@ def parse_fa_report_document(fa_report_document: ET.Element):
     else:
         data = dict()
 
-    fa_report_document_data = {**metadata, **data}
+    if activity_datetimes_utc:
+        activity_datetime_utc = min(activity_datetimes_utc)
+    else:
+        activity_datetime_utc = None
+
+    fa_report_document_data = {
+        "activity_datetime_utc": activity_datetime_utc,
+        **metadata,
+        **data,
+    }
 
     return fa_report_document_data
 
@@ -274,7 +285,28 @@ def parse_specified_fishing_activity(
             f"Could not find appropriate parser for log type {log_type}: ", e
         )
     value = parser(fishing_activity)
-    return log_type, value
+
+    datetime_string = get_text(
+        fishing_activity, ".//ram:OccurrenceDateTime/udt:DateTime"
+    )
+    if not datetime_string:
+        datetime_string = get_text(
+            fishing_activity,
+            "./ram:SpecifiedDelimitedPeriod/ram:EndDateTime/udt:DateTime",
+        )
+
+    if datetime_string:
+        try:
+            activity_datetime_utc = parse(datetime_string).replace(tzinfo=None)
+        except Exception as e:
+            logging.error(
+                f"Cound not parse datetime string {datetime_string} with error: {e}"
+            )
+            activity_datetime_utc = None
+    else:
+        activity_datetime_utc = None
+
+    return log_type, activity_datetime_utc, value
 
 
 def get_list_fa_report_documents(fa_report_message: ET.Element) -> List[ET.Element]: