Initial add reports

kolzchut · Oct 1, 2024 · 487b763 · 487b763
1 parent 6da44fc
commit 487b763
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 2 deletions.
diff --git a/operators/derive/to_dp.py b/operators/derive/to_dp.py
@@ -7,6 +7,7 @@
 from thefuzz import fuzz
 
 from conf import settings
+from srm_tools.stats import Report
 from .autocomplete import IGNORE_SITUATIONS
 from srm_tools.logger import logger
 from srm_tools.unwind import unwind
@@ -622,6 +623,13 @@ def func(ids):
 
     rs_score = RSScoreCalc()
 
+    invalid_location_report = Report(
+        'Processing: Cards: Invalid Location Report',
+        'cards-invalid-location',
+        ['branch_id', 'organization_id', 'organization_name', 'branch_address', 'branch_geometry'],
+        ['branch_id']
+    )
+
     return DF.Flow(
         DF.checkpoint(CHECKPOINT),
         DF.add_field('situations', 'array', lambda r: [situations[s] for s in r['situation_ids']], resources=['card_data']),
@@ -653,7 +661,12 @@ def func(ids):
         ),
         helpers.get_stats().filter_with_stat('Processing: Cards: No Response Category', lambda r: r['response_category'], resources=['card_data']),
         DF.set_type('responses', transform=lambda v, row: helpers.reorder_responses_by_category(v, row['response_category'])),
-        helpers.get_stats().filter_with_stat('Processing: Cards: Invalid Location',lambda r: helpers.validate_geometry(r['branch_geometry']) or r['national_service'], resources=['card_data']),
+        helpers.get_stats().filter_with_stat(
+            'Processing: Cards: Invalid Location',
+            lambda r: helpers.validate_geometry(r['branch_geometry']) or r['national_service'],
+            resources=['card_data'],
+            report=invalid_location_report
+        ),
         DF.add_field('possible_autocomplete', 'array', default=possible_autocomplete, resources=['card_data'], **{'es:itemType': 'string', 'es:keyword': True}),
         DF.add_field(
             'point_id', 'string',

diff --git a/srm_tools/stats.py b/srm_tools/stats.py
@@ -1,9 +1,39 @@
 import dataflows as DF
 from dataflows.helpers.resource_matcher import ResourceMatcher
 from dataflows_airtable import load_from_airtable, dump_to_airtable, AIRTABLE_ID_FIELD
+from dataflows_ckan import dump_to_ckan
 
 from conf import settings
 
+class Report():
+
+    def __init__(self, name, slug, fields, id_fields):
+        self.name = name
+        self.slug = slug
+        self.fields = fields
+        self.id_fields = id_fields
+        self.added = set()
+        self.records = []
+
+    def add(self, rec):
+        key = tuple(rec[f] for f in self.id_fields)
+        if key in self.added:
+            return
+        self.added.add(key)
+        self.records.append([rec.get(f) for f in self.fields])
+
+    def save(self):
+        DF.Flow(
+            [self.records],
+            DF.update_package(title=self.name, name=self.slug),
+            DF.update_resource(-1, name='report', path='report.csv'),
+            dump_to_ckan(
+                settings.CKAN_HOST,
+                settings.CKAN_API_KEY,
+                settings.CKAN_OWNER_ORG,
+            )
+        ).process()
+
 class Stats():
 
     def __init__(self):
@@ -49,17 +79,21 @@ def save(self):
             self.set_stat(stat, value)
         self.dirty = {}
 
-    def filter_with_stat(self, stat: str, filter_func, passing=False, resources=None):
+    def filter_with_stat(self, stat: str, filter_func, passing=False, resources=None, report: Report=None):
 
         def process_resource(rows):
             count = 0
             for row in rows:
                 if filter_func(row):
                     if passing:
+                        if report:
+                            report.add(row)
                         count += 1
                     yield row
                 else:
                     if not passing:
+                        if report:
+                            report.add(row)
                         count += 1
             self.set_stat(stat, count)