Bus Cost Analysis Refactor 2

bus_procurement_cost/Makefile

@@ -1,8 +1,15 @@
 # runs all scripts for bus procurement cost
 all_bus_scripts:
-	python fta_data_cleaner.py
-	python tircp_data_cleaner.py
-	python dgs_data_cleaner.py
-	python cost_per_bus_cleaner.py
-	jupyter nbconvert --to notebook --execute --inplace cost_per_bus_analysis.ipynb
-	jupyter nbconvert --to html --no-input --no-prompt cost_per_bus_analysis.ipynb
+	python _01_fta_data_cleaner.py
+	python _02_tircp_data_cleaner.py
+	python _03_dgs_data_cleaner.py
+	python _04_cost_per_bus_cleaner.py
+
+	#execute NB
+	jupyter nbconvert --to notebook --execute --inplace _05_cost_per_bus_analysis.ipynb
+
+	#convert NB to HTML then to PDF
+	jupyter nbconvert --to html --no-input --no-prompt _05_cost_per_bus_analysis.ipynb
+	pip install WeasyPrint
+	weasyprint _05_cost_per_bus_analysis.html cost_per_bus_analysis.pdf
+

bus_procurement_cost/README.md

@@ -57,28 +57,28 @@ Analyze bus procurement projects to see how much transit agencies pay for them.
 Executing `make all_bus_scripts` will run the following scripts
 <br></br>
 
-- **fta_data_cleaner.py:**
+- **_01_fta_data_cleaner.py:**
     * Reads in and cleans FTA data
     * outputs 2 files: 
         * cleaned, all projects: `clean_fta_all_projects.parquet`
         * cleaned, bus only projects:`clean_fta_bus_only.parquet`
 <br></br>        
 
-- **tircp_data_cleaner.py**
+- **_02_tircp_data_cleaner.py**
     * Reads in and cleans tircp data
     * outputs 2 files: 
         * cleaned, all projects: `clean_tircp_all_project.parquet`
         * cleaned, bus only projects:`clean_tircp_bus_only.parquet`
 <br></br>
 
-- **dgs_data_cleaner.py**
+- **_03_dgs_data_cleaner.py**
     * Reads in and cleans DGS data
     * outputs 2 files: 
         * cleaned, bus only projects: `clean_dgs_all_projects.parquet`
         * cleaned, bus only projects with options:`clean_dgs_bus_only_w_options.parquet`
 <br></br>
 
-- **cost_per_bus_cleaner.py**
+- **_04_cost_per_bus_cleaner.py**
     * Reads in and merges all the bus only datasets
     * updates columns names
     * calculates `cost_per_bus`, z-score and idetifies outliers.
@@ -97,4 +97,9 @@ Executing `make all_bus_scripts` will run the following scripts
     * hides the code cells and prompts
 <br></br>
 
-output files are saved to GCS at: `calitp-analytics-data/data-analyses/bus_procurement_cost`
+- **weasyprint ...html ...pdf
+    * convers the HTML files to PDF, perserving the same style fonts, tables and charts.
+
+Output data files are saved to GCS at: `calitp-analytics-data/data-analyses/bus_procurement_cost`
+
+Final deliverable: `cost_per_bus_analysis.pdf`

bus_procurement_cost/fta_data_cleaner.py → bus_procurement_cost/_01_fta_data_cleaner.py

@@ -2,7 +2,7 @@
 import pandas as pd
 import shared_utils
 from calitp_data_analysis.sql import to_snakecase
-from bus_cost_utils import *
+from _bus_cost_utils import GCS_PATH, new_prop_finder, new_bus_size_finder, project_type_finder, col_row_updater
 
 def col_splitter(
     df: pd.DataFrame, 

bus_procurement_cost/tircp_data_cleaner.py → bus_procurement_cost/_02_tircp_data_cleaner.py

@@ -2,7 +2,7 @@
 import pandas as pd
 import shared_utils
 from calitp_data_analysis.sql import to_snakecase
-from bus_cost_utils import *
+from _bus_cost_utils import GCS_PATH, new_prop_finder, new_bus_size_finder, project_type_finder, col_row_updater
 
 def clean_tircp_columns() -> pd.DataFrame:
     """

bus_procurement_cost/dgs_data_cleaner.py → bus_procurement_cost/_03_dgs_data_cleaner.py

@@ -2,7 +2,7 @@
 import pandas as pd
 import shared_utils
 from calitp_data_analysis.sql import to_snakecase
-from bus_cost_utils import *
+from _bus_cost_utils import GCS_PATH, new_prop_finder, new_bus_size_finder
 
 def calculate_total_cost(row):
     """

bus_procurement_cost/cost_per_bus_cleaner.py → bus_procurement_cost/_04_cost_per_bus_cleaner.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from bus_cost_utils import *
+from _bus_cost_utils import GCS_PATH, new_outlier_flag_v2
 from scipy.stats import zscore
 
 
@@ -81,7 +81,7 @@ def prepare_all_data() ->pd.DataFrame:
     merge2["zscore_cost_per_bus"] = zscore(merge2["cost_per_bus"])
 
     #flag any outliers
-    merge2["is_cpb_outlier?"] = merge2["zscore_cost_per_bus"].apply(outlier_flag)
+    merge2["is_cpb_outlier?"] = new_outlier_flag_v2(merge2,'zscore_cost_per_bus')
     return merge2