reorganize datascripts #34

Author: robindemourat

.gitignore

@@ -32,4 +32,7 @@ yarn-error.log*
 
 
 # unwanted tests for now
-/src/components/LinearGraphBlock/
+/src/components/LinearGraphBlock/
+
+
+__pycache__

datascripts/decline.py

@@ -0,0 +1,41 @@
+
+import requests
+import csv
+import os
+import coloredlogs, logging
+
+logger = logging.getLogger(__name__)
+coloredlogs.install(level='DEBUG')
+
+def get_online_csv(url):
+  """
+  Cette fonction permet de récupérer le contenu d'un csv en ligne.
+  Pour les google spreadsheets: fichier > publier sur le web > format csv > copier le lien
+  """
+  results = []
+  with requests.Session() as s:
+      download = s.get(url)
+      decoded_content = download.content.decode('utf-8')
+      reader = csv.DictReader(decoded_content.splitlines(), delimiter=',')
+      for row in reader:
+        results.append(row)
+  return results
+
+
+def ensure_dir(path):
+  if not os.path.exists(path):
+      os.makedirs(path)
+
+def write_csv(filename, data):
+  logger.debug('write csv | ' + filename)
+  parts = filename.split('/')
+  if len(parts) > 1:
+    folder = parts[0]
+    folder_path = "../public/data/" + folder
+    ensure_dir(folder_path)
+  final_path = "../public/data/" + filename;
+  with open(final_path, "w") as of:
+    output_csv = csv.DictWriter(
+        of, data[0].keys())
+    output_csv.writeheader()
+    output_csv.writerows(data)

datascripts/lib.py

@@ -0,0 +1,110 @@
+from collections import defaultdict
+import csv
+from typing import Counter, DefaultDict
+import os
+from lib import ensure_dir, logger
+
+
+logger.info('start | part 1 main viz datasets')
+
+def output_row(region, year, region_trade, region_products, total_trade):
+  sum_imports = sum(value.get(
+      'Imports') for value in region_products[year].values() if value.get('Imports'))
+  sum_exports = sum(value.get(
+      'Exports') for value in region_products[year].values() if value.get('Exports'))
+  return {
+      'region': region,
+      'year': year,
+      'Exports': region_trade[year].get('Exports'),
+      'Imports': region_trade[year].get('Imports'),
+      'Imports_share': region_trade[year]['Imports']/total_trade['Imports'] if region_trade[year].get('Imports') else None,
+      'Exports_share': region_trade[year]['Exports']/total_trade['Exports'] if region_trade[year].get('Exports') else None,
+      'product_revolutionempire_imports_herfindahl': sum(pow(value['Imports']/sum_imports, 2) for value in region_products[year].values()) if sum_imports != 0 else None,
+      'product_revolutionempire_exports_herfindahl': sum(pow(value['Exports']/sum_exports, 2) for value in region_products[year].values()) if sum_exports != 0 else None,
+      'product_revolutionempire_total_herfindahl': sum(pow((value['Imports']+value['Exports'])/(sum_imports + sum_exports), 2) for value in region_products[year].values()) if sum_imports != 0 or sum_exports != 0 else None
+  }
+
+
+with open('../data/toflit18_all_flows.csv', 'r') as f:
+  toflit18_flows = csv.DictReader(f)
+  flows_fieldnames = toflit18_flows.fieldnames
+
+  france_trade = DefaultDict(Counter)
+  LaRochelle_trade = DefaultDict(Counter)
+  Bordeaux_trade = DefaultDict(Counter)
+
+  Bordeaux_products = DefaultDict(
+      lambda: DefaultDict(Counter))
+  LaRochelle_products = DefaultDict(
+      lambda: DefaultDict(Counter))
+  LaRochelle_partners = DefaultDict(
+      lambda: DefaultDict(Counter))
+  for flow in toflit18_flows:
+      # ATTENTION we filter out Ports Francs
+      if flow['partner_grouping'] == 'France':
+          continue
+      # longitudinal absolute and share trade
+      year = flow['year'].split(
+          '.')[0] if "." in flow['year'] else flow['year']
+      if flow['best_guess_national_partner'] == "1" and flow['value'] != "":
+          france_trade[year][flow['export_import']
+                              ] += float(flow['value'])
+      if flow['best_guess_region_prodxpart'] == "1" and flow['value'] != "":
+          try:
+
+              if flow['customs_region'] == "La Rochelle":
+                  LaRochelle_trade[year
+                                    ][flow['export_import']] += float(flow['value'])
+              if flow['customs_region'] == "Bordeaux":
+                  Bordeaux_trade[year
+                                  ][flow['export_import']] += float(flow['value'])
+          except ValueError as e:
+              logger.warning(flow['value'])
+          except KeyError as e:
+              pass
+      # product and partner top for La Rochelle
+      if flow['best_guess_region_prodxpart'] == "1" and flow['value'] != "":
+          if flow['customs_region'] == "La Rochelle":
+              LaRochelle_products[year][flow['product_revolutionempire']
+                                        ][flow['export_import']] += float(flow['value'])
+              LaRochelle_partners[year
+                                  ][flow['partner_simplification']][flow['export_import']] += float(flow['value'])
+          if flow['customs_region'] == "Bordeaux":
+              Bordeaux_products[year][flow['product_revolutionempire']
+                                      ][flow['export_import']] += float(flow['value'])
+  ensure_dir("../public/data/decline_longitudinal_data")
+  with open("../public/data/decline_longitudinal_data/decline_longitudinal_data.csv", "w") as of:
+      output_csv = csv.DictWriter(
+          of, ['region', 'year', 'Exports', 'Imports', 'Exports_share', 'Imports_share', 'product_revolutionempire_imports_herfindahl', 'product_revolutionempire_exports_herfindahl', 'product_revolutionempire_total_herfindahl'])
+      output_csv.writeheader()
+      for year, value in sorted(france_trade.items(), key=lambda yv: yv[0]):
+
+          output_csv.writerow(output_row(
+              "La Rochelle", year, LaRochelle_trade, LaRochelle_products, value))
+          output_csv.writerow(output_row(
+              "Bordeaux", year, Bordeaux_trade, Bordeaux_products, value))
+          output_csv.writerow({
+              'region': 'France',
+              'year': year,
+              'Exports': value['Exports'],
+              'Imports': value['Imports'],
+              'Imports_share': 100,
+              'Exports_share': 100
+          })
+  ensure_dir("../public/data/decline_LR_products")
+  with open("../public/data/decline_LR_products/decline_LR_products.csv", "w") as of:
+      output_csv = csv.DictWriter(
+          of, ['product', 'year', 'Exports', 'Imports'])
+      output_csv.writeheader()
+      output_csv.writerows({'product': product, 'year': year, 'Exports': value.get("Exports"), 'Imports': value.get("Imports")} for year, products in LaRochelle_products.items(
+      ) if year in ['1750', '1789'] for product, value in products.items())
+
+  ensure_dir("../public/data/decline_LR_partners")
+  with open("../public/data/decline_LR_partners/decline_LR_partners.csv", "w") as of:
+      output_csv = csv.DictWriter(
+          of, ['partner', 'year', 'Exports', 'Imports'])
+      output_csv.writeheader()
+      output_csv.writerows({'partner': partner, 'year': year, 'Imports': value.get('Imports'), 'Exports': value.get('Exports')} for year, partners in LaRochelle_partners.items(
+      ) if year in ['1750', '1789'] for partner, value in partners.items())
+
+logger.debug('done | part 1 main viz datasets')

datascripts/part_1.py

@@ -8,15 +8,9 @@
 '''
 
 import csv
-import sys
-from collections import defaultdict
 import os
+from lib import ensure_dir, logger
 
-def ensure_dir(path):
-  if not os.path.exists(path):
-      os.makedirs(path)
-
-OUTPUT = "../public/data/part_2_navigo_viz_data/part_2_navigo_viz_data.csv"
 ensure_dir("../public/data/part_2_navigo_viz_data/")
 
 def clean_bureau_name(name, departure):
@@ -46,10 +40,6 @@ def clean_bureau_name(name, departure):
       if flow['departure_ferme_direction'] == 'La Rochelle' and flow['departure_function'] == 'O':
         relevant_flows.append(flow)
 
-
-#print(sys.getdefaultencoding())
-#print(sys.stdout.encoding)
-
 for f in relevant_flows :
     destination_radar='Unknown'
     if f['destination_partner_balance_supp_1789']=='Sénégal et Guinée':
@@ -71,8 +61,8 @@ def clean_bureau_name(name, departure):
 
 
     if (destination_radar=='Unknown'):
-        print(f['destination_partner_balance_supp_1789'])
-        print(f['destination_partner_balance_supp_1789'].encode("utf8"))
+        logger.warning('unknown radar destination : ' + f['destination_partner_balance_supp_1789'])
+        logger.warning('unknown radar destination : ' + f['destination_partner_balance_supp_1789'].encode("utf8"))
     #Create and assign a new column named destination_radar
     f['destination_radar'] = destination_radar
 
@@ -103,8 +93,8 @@ def clean_bureau_name(name, departure):
 
     #Check all is assigned
     if (homeport_destination_radar=='Unknown'):
-        print(f['homeport_substate_1789_fr'])
-        print(f['homeport_substate_1789_fr'].encode("utf8"))
+        logger.warning('unknown homeport destination radar : ' + f['homeport_substate_1789_fr'])
+        logger.warning('unknown homeport destination radar : ' + f['homeport_substate_1789_fr'].encode("utf8"))
     #Create and assign a new column named homeport_destination_radar
     f['homeport_destination_radar'] = homeport_destination_radar
 
@@ -131,11 +121,15 @@ def format_for_viz(f):
    }
 
 initial_flows_viz = [format_for_viz(f) for f in relevant_flows]
+
 # write dataset
-with open(OUTPUT, "w", newline='') as csvfile:
+destination_filepath = "../public/data/part_2_navigo_viz_data/part_2_navigo_viz_data.csv"
+with open(destination_filepath, "w", newline='') as csvfile:
+  logger.info('start | part 2 main viz navigo data')
   fieldnames = initial_flows_viz[0].keys()
   writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
 
   writer.writeheader()
   for f in initial_flows_viz:
       writer.writerow(f)
+  logger.debug('done | part 2 main viz navigo data')

datascripts/retrieve_part_2_main_viz_navigo_for_radar.py renamed to datascripts/part_2_navigo.py

@@ -1,14 +1,10 @@
 import csv
 from collections import defaultdict
 import os
+from lib import ensure_dir, logger
 
-def ensure_dir(path):
-  if not os.path.exists(path):
-      os.makedirs(path)
-
-OUTPUT = "../public/data/part_2_toflit_viz_data/part_2_toflit_viz_data.csv"
-ensure_dir("../public/data/part_2_toflit_viz_data/")
 
+logger.info('start | part 2 main viz toflit18 data')
 relevant_flows = []
 # retrieve relevant flows
 with open('../data/toflit18_all_flows.csv', 'r') as f:
@@ -30,7 +26,6 @@ def ensure_dir(path):
 for f in relevant_flows :
     product_weight_kg = 0
     # @todo a lot of products are flushed out when doing thing
-    # print(f['quantities_metric'], f['quantity_unit_metric'])
     if f['quantity_unit_metric'] and f['quantity_unit_metric'] == 'kg':
       product_weight_kg = float(f['quantities_metric'] if f['quantities_metric'] else 0)
     f['product_weight_kg'] = product_weight_kg
@@ -108,10 +103,13 @@ def format_for_viz(f):
 flows_viz = list(uniques.values())
 
 # write dataset
-with open(OUTPUT, "w") as csvfile:
+dataset_filepath = "../public/data/part_2_toflit_viz_data/part_2_toflit_viz_data.csv"
+ensure_dir("../public/data/part_2_toflit_viz_data/")
+with open(dataset_filepath, "w") as csvfile:
   fieldnames = flows_viz[0].keys()
   writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
 
   writer.writeheader()
   for f in flows_viz:
-      writer.writerow(f)
+      writer.writerow(f)
+  logger.debug('done | part 2 main viz toflit18 data')