Merge pull request #21 from oislen/dev

12-webscraper

.dockerignore

@@ -1,4 +1,5 @@
 data/Met_Eireann/arch
+data/Met_Eireann/scraped_data
 *__pycache__
 *.ipynb_checkpoints
 *.xlsx

.github/workflows/unittest-dev.yml

@@ -17,6 +17,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Set environment variable
+      run: echo "GITHUB_ACTIONS_UNITTEST_FLAG=1" >> $GITHUB_ENV
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:

.github/workflows/unittest-main-pr.yml

@@ -15,6 +15,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Set environment variable
+      run: echo "GITHUB_ACTIONS_UNITTEST_FLAG=1" >> $GITHUB_ENV
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:

.github/workflows/unittest-main-push.yml

@@ -15,6 +15,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Set environment variable
+      run: echo "GITHUB_ACTIONS_UNITTEST_FLAG=1" >> $GITHUB_ENV
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:

.gitignore

@@ -1,5 +1,6 @@
 # ignore sub repos
 data/Met_Eireann/arch
+data/Met_Eireann/scraped_data
 *__pycache__
 *.ipynb_checkpoints
 *.xlsx

conda/irishclimatedashboard.yml

@@ -32,8 +32,8 @@ dependencies:
     - black==24.10.0
     - bleach==6.1.0
     - bokeh==3.6.0
-    - boto3==1.35.37
-    - botocore==1.35.37
+    - boto3==1.35.38
+    - botocore==1.35.38
     - certifi==2024.8.30
     - cffi==1.17.1
     - chardet==5.2.0
@@ -44,7 +44,7 @@ dependencies:
     - comm==0.2.2
     - contourpy==1.3.0
     - cycler==0.12.1
-    - debugpy==1.8.6
+    - debugpy==1.8.7
     - decorator==5.1.1
     - defusedxml==0.7.1
     - et-xmlfile==1.1.0

dashboard/bokeh_dash_app.py

@@ -1,18 +1,17 @@
 # load relevant libraries
-import cons
 from bokeh.io import curdoc
 from bokeh.models import TabPanel, Tabs
 
 # load custom modules
-from BokehApp.bokeh_line_dash import bokeh_line_dash
-from BokehApp.bokeh_map_dash import bokeh_map_dash
+from utilities.bokeh_line_dash import bokeh_line_dash
+from utilities.bokeh_map_dash import bokeh_map_dash
 
 # initialise and structure bokeh line dashboard
-dashboard_line = bokeh_line_dash(load_data_dict = cons.load_data_dict)
+dashboard_line = bokeh_line_dash()
 panel_line_tab = TabPanel(child = dashboard_line, title = 'Time Series')
 
 # initialise and structure bokeh map dashboard
-dashboard_map = bokeh_map_dash(load_data_dict = cons.load_data_dict)
+dashboard_map = bokeh_map_dash()
 panel_map_tab = TabPanel(child = dashboard_map, title = 'GIS Map')
 
 # create combined dashboard

dashboard/cons.py

@@ -1,77 +1,66 @@
 # load relevant libraries
 import os
+import sys
 import platform
+from datetime import datetime
+import json
 
-root_dir = 'E:\\GitHub\\IrishClimateDashboard' if platform.system() == 'Windows' else '/home/ubuntu/IrishClimateDashboard'
+root_dir = 'E:\\GitHub\\IrishClimateDashboard' if platform.system() == 'Windows' else "/home/runner/work/IrishClimateDashboard/IrishClimateDashboard" if "GITHUB_ACTIONS_UNITTEST_FLAG" in os.environ else '/home/ubuntu/IrishClimateDashboard'
+sys.path.append(root_dir)
 # set directories
 data_dir = os.path.join(root_dir, 'data')
-census_2011_dir = os.path.join(data_dir, 'Census_2011')
-census_2016_dir = os.path.join(data_dir, 'Census_2016')
-met_eireann_dir = os.path.join(data_dir, 'Met_Eireann')
-gis_dir = os.path.join(data_dir, "gis")
+bokeh_ref_data_dir = os.path.join(data_dir, "bokeh", "ref")
 # set data files
 master_data_fpath = os.path.join(data_dir, 'master.feather')
+gis_dir = os.path.join(data_dir, "gis")
 preaggregate_data_fpath = os.path.join(data_dir, "preaggregate_data.pickle")
 bokeh_line_data_fpath = os.path.join(data_dir, "bokeh_line_data.pickle")
 bokeh_map_data_fpath = os.path.join(data_dir, "bokeh_map_data.pickle")
-rep_counties_fpath = os.path.join(data_dir, 'gis', 'arch', 'Counties_-_OSi_National_Statutory_Boundaries_-_2019', 'Counties___OSi_National_Statutory_Boundaries_.shp')
-ni_counties_fpath = os.path.join(data_dir, 'gis', 'arch', 'northern_ireland_counties', 'northern_ireland_counties.shp')
-counties_data_fpath = os.path.join(gis_dir, "counties.shp")
 map_data_fpath = os.path.join(gis_dir, "map_data.pickle")
 points_data_fpath = os.path.join(gis_dir, "points_data.pickle")
-stations_fpath = os.path.join(met_eireann_dir, 'ref', 'StationDetails.csv')
-
-# met eireann weather data
-met_eireann = {'dublin_airport':'https://cli.fusio.net/cli/climate_data/webdata/dly532.zip'}
+# set bokeh reference data file paths
+county_line_colors_fpath = os.path.join(bokeh_ref_data_dir, "county_line_colors.json")
+map_settings_fpath = os.path.join(bokeh_ref_data_dir, "map_settings.json")
+figure_settings_fpath = os.path.join(bokeh_ref_data_dir, "fig_settings.json")
+unittest_normal_dists_fpath = os.path.join(bokeh_ref_data_dir, "unittest_normal_dists.json")
+col_options_fpath = os.path.join(bokeh_ref_data_dir, "col_options.json")
+stat_options_fpath = os.path.join(bokeh_ref_data_dir, "stat_options.json")
+agg_level_strftime_fpath = os.path.join(bokeh_ref_data_dir, "agg_level_strftime.json")
 
 # seaborn plot settings
 sns_fig_settings = {'figure.figsize':(7, 7), "lines.linewidth": 0.7}
 
 # set whether to load in pickle data for bokeh app or generate from master file
 load_data_dict = True
 
-# bokeh figure settings
-FIG_SETTING = {'height':640, 
-               'width':640, 
-               'min_border_left':40, 
-               'min_border_right':40,
-               'min_border_top':40, 
-               'min_border_bottom': 16, 
-               'tools':'pan,wheel_zoom, box_zoom,reset,save'
-               }
-
-# bokeh map settings
-MAP_SETTINGS = {'line_color':'gray', 
-                'line_width':0.25, 
-                'fill_alpha':1
-                }
+# load bokeh reference data
+with open(county_line_colors_fpath) as json_file: 
+    county_line_colors = json.load(json_file)
+with open(map_settings_fpath) as json_file: 
+    MAP_SETTINGS = json.load(json_file)
+with open(figure_settings_fpath) as json_file: 
+    FIG_SETTING = json.load(json_file)
+with open(col_options_fpath) as json_file: 
+    col_options = json.load(json_file)
+with open(stat_options_fpath) as json_file: 
+    stat_options = json.load(json_file)
+with open(agg_level_strftime_fpath) as json_file: 
+    date_strftime_dict = json.load(json_file)
 
 # bokeh line selector settings
-county_line_colors = {'Donegal':'greenyellow', 'Dublin':'lightblue', 
-                      'Carlow':'sandybrown', 'Cavan':'dodgerblue', 'Clare':'yellow', 'Cork':'red', 
-                      'Galway':'maroon', 
-                      'Kerry':'gold', 'Kildare':'lavender', 'Kilkenny':'black', 
-                      'Laois':'fuchsia', 'Leitrim':'khaki', 'Limerick':'olivedrab', 'Longford':'blueviolet',
-                      'Mayo':'green',  'Meath':'lawngreen', 'Monaghan':'slateblue',
-                      'Offaly':'lightgreen',
-                      'Roscommon':'cornsilk',
-                      'Sligo':'indianred', 
-                      'Tipperary':'royalblue',
-                      'Waterford':'whitesmoke', 'Westmeath':'darkred', 'Wexford':'purple', 'Wicklow':'darkblue'
-                      }
-
 counties = list(county_line_colors.keys())
 line_colors = list(county_line_colors.values())
 counties_values = [str(i) for i in range(len(counties))]
 counties_options = [(str(i), c) for i, c in enumerate(counties)]
-date_strftime_dict = {'year':'%Y', 'year-month':'%Y-%m', 'month':'%m'}
 line_agg_level_options = list(date_strftime_dict.keys())
 line_agg_level_default = line_agg_level_options[0]
-col_options = ['maxtp', 'mintp', 'gmin', 'soil', 'wdsp', 'sun', 'evap', 'rain']
 col_default = col_options[0]
-stat_options = ['mean', 'median', 'max', 'min', 'var', 'std', 'sum']
 stat_default = stat_options[0]
 show_stations_default = []
+linedash_yearend = str(int(datetime.now().strftime('%Y')) - 1)
+linedash_year_timespan = ["2010", linedash_yearend]
+linedash_yearmonth_timespan = ["2010-01", f"{linedash_yearend}-12"]
+linedash_month_timespan = ["01", "12"]
 
 # bokeh server execution commands
 bat_execBokehApp = "START /MIN CMD.EXE /C exeBokehApp.bat"
@@ -81,23 +70,5 @@
 unittest_n_dates = 3
 unittest_country_station_map ={'dublin':['dublin airport', 'casement'], 'cork':['ucc']}
 unittest_start_date = '2020-01-01'
-unittest_normal_dists = {
-    "maxtp":{"loc":13, "scale":4.9},
-    "mintp":{"loc":6.1, "scale":4.3},
-    "igmin":{"loc":0.2, "scale":0.4},
-    "gmin":{"loc":.4, "scale":5.2},
-    "rain":{"loc":2.0, "scale":4.4},
-    "wdsp":{"loc":9.9, "scale":4},
-    "hm":{"loc":17, "scale":6},
-    "ddhm":{"loc":206, "scale":85},
-    "hg":{"loc":25, "scale":8.8},
-    "sun":{"loc":4.1, "scale":3.8},
-    "dos":{"loc":0.3, "scale":18},
-    "glorad":{"loc":1062, "scale":741},
-    "soil":{"loc":11.2, "scale":5.3},
-    "pe":{"loc":1.6, "scale":1.0},
-    "evap":{"loc":2.3, "scale":1.4},
-    "smd_wd":{"loc":20.5, "scale":22.5},
-    "smd_md":{"loc":19.9, "scale":23.1},
-    "smd_pd":{"loc":18.2, "scale":27.3}
-}
+with open(unittest_normal_dists_fpath) as json_file: 
+    unittest_normal_dists = json.load(json_file)

dashboard/unittests/utilities/test_time_data.py

@@ -19,8 +19,8 @@
 # generate unittest data
 data = gen_unittest_data()
 obs_time_data = time_data(data=data, agg_dict=agg_dict)
-exp_data_shape = (6, 12)
-exp_data_columns = ['county', 'date', 'date_str', 'maxtp', 'mintp', 'gmin', 'soil', 'wdsp', 'sun', 'evap', 'rain', 'index']
+exp_data_shape = (6, 13)
+exp_data_columns = ['county', 'date', 'date_str', 'maxtp', 'mintp', 'gmin', 'soil', 'wdsp', 'sun', 'evap', 'rain', 'glorad', 'index']
 
 class Test_time_data(unittest.TestCase):
     """"""

dashboard/BokehApp/bokeh_line_dash.py → dashboard/utilities/bokeh_line_dash.py

@@ -1,36 +1,28 @@
 import pickle
+from beartype import beartype
+from typing import Union
 from bokeh.models import Select, CheckboxGroup, Div, Button, MultiSelect
 from bokeh.layouts import column, row
 
 # import custom modules
 import cons
-from PreProcessData.gen_preaggregate_data import gen_preaggregate_data
-from BokehApp.bokeh_line_data import bokeh_line_data
-from BokehApp.bokeh_line_plot import bokeh_line_plot
-from beartype import beartype
-from typing import Union
+from utilities.bokeh_line_data import bokeh_line_data
+from utilities.bokeh_line_plot import bokeh_line_plot
 
 @beartype
-def bokeh_line_dash(
-    load_data_dict:bool=True
-    ):
+def bokeh_line_dash():
     """Generates the bokeh line dashboard
 
     Parameters
     ----------
-    load_data_dict : bool
-        Whether to load the preaggregated file from disk, or generate it scratch, default is True
-
+    
     Returns
     -------
     bokeh.layouts.row
         The interactive bokeh line dashboard
     """
-    if load_data_dict:
-        with open(cons.preaggregate_data_fpath, "rb") as handle:
-            pre_agg_data_dict = pickle.load(handle)
-    else:
-        pre_agg_data_dict = gen_preaggregate_data(return_data=True)
+    with open(cons.preaggregate_data_fpath, "rb") as handle:
+        pre_agg_data_dict = pickle.load(handle)
     # generate bokeh data for line plot
     bokeh_line_data_dict = bokeh_line_data(pre_agg_data_dict)
     # create bokeh plot