ICF Off-model calculator - Refactor 1-3, Feature 1-3

utilities/RTP/Emissions/Off Model Calculators/data/input/IPA_TM2/ModelData/Model Data - Bikeshare.csv

@@ -0,0 +1,38 @@
+Output by X:/travel-model-one-master/utilities/RTP/Emissions/Off Model Calculators/BikeShare.R on Thu Feb 01 10:43:17 2024
+"index","year","category","directory","variable","value"
+"2005-SB375 Base year-total_employment",2005,"SB375 Base year","2005_TM160_IPA_02","total_employment",3575933
+"2005-SB375 Base year-total_population",2005,"SB375 Base year","2005_TM160_IPA_02","total_population",7096469
+"2015-Previous base year-total_employment",2015,"Previous base year","2015_TM160_IPA_03","total_employment",3861318
+"2015-Previous base year-total_population",2015,"Previous base year","2015_TM160_IPA_03","total_population",7581396
+"2023-Base year-total_employment",2023,"Base year","2023_TM160_IPA_42","total_employment",4064307
+"2023-Base year-total_population",2023,"Base year","2023_TM160_IPA_42","total_population",7847766
+"2025-Previous Plan-total_employment",2025,"Previous Plan","2025_TM152_FBP_Plus_22","total_employment",4147691
+"2025-Previous Plan-total_population",2025,"Previous Plan","2025_TM152_FBP_Plus_22","total_population",8231265
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_09","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_09_minusModePref","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_09_PBA50aoc","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_09_PBA50ixex","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_09_PBA50landuse","total_employment",4834513
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_10","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_10_plusEN7","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_11","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_11_network2023","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_12","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_14","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_15","total_employment",4854742
+"2035-IPA-total_employment",2035,"IPA","2035_TM160_IPA_16","total_employment",4854742
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_09","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_09_minusModePref","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_09_PBA50aoc","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_09_PBA50ixex","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_09_PBA50landuse","total_population",9002950
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_10","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_10_plusEN7","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_11","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_11_network2023","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_12","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_14","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_15","total_population",8476332
+"2035-IPA-total_population",2035,"IPA","2035_TM160_IPA_16","total_population",8476332
+"2035-Previous Plan-total_employment",2035,"Previous Plan","2035_TM152_FBP_Plus_24","total_employment",4834513
+"2035-Previous Plan-total_population",2035,"Previous Plan","2035_TM152_FBP_Plus_24","total_population",9002950

utilities/RTP/Emissions/Off Model Calculators/helper/bshare.py

@@ -0,0 +1,54 @@
+import openpyxl
+
+from helper.calcs import Calc
+
+class Bikeshare(Calc):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.masterWbName="PBA50+_OffModel_Bikeshare"
+        self.dataFileName="Model Data - Bikeshare"
+
+    def write_runid_to_mainsheet(self):
+        # get variables location in calculator
+        Calc.get_variable_locations(self)
+
+        # add run_id to 'Main sheet'
+        newWorkbook = openpyxl.load_workbook(self.new_workbook_file)
+        mainsheet = newWorkbook['Main sheet']
+
+        # Select Main sheet variables
+        vMS=self.v['Main sheet']
+
+        # Write run name and year
+        mainsheet[vMS['Run_directory_2035']] = Calc.get_ipa(self, self.runs[0])[0]
+        mainsheet[vMS['Run_directory_2050']] = Calc.get_ipa(self, self.runs[1])[0]
+        mainsheet[vMS['year_a']] = Calc.get_ipa(self, self.runs[0])[1]
+        mainsheet[vMS['year_b']] = Calc.get_ipa(self, self.runs[1])[1]
+
+
+        # save file
+        newWorkbook.save(self.new_workbook_file)
+
+        if self.verbose:
+            print(f"Main sheet updated with {self.runs} in location\n{self.new_workbook_file}")
+
+    def update_calculator(self):
+
+        # Step 1: Create run and copy files  
+        Calc.copy_workbook(self)
+
+        # Step 2: load and filter model data of selected runs
+        modelData, metaData=Calc.get_model_data(self)
+
+        # Step 3: add model data of selected runs to 'Model Data' sheet
+        Calc.write_model_data_to_excel(self,modelData,metaData)
+
+        # Step 4:
+        self.write_runid_to_mainsheet()
+
+        ## Step 5: open/close Excel, autosave
+        # todo
+        # 
+        # Step 6: log runs in master
+        # todo   

utilities/RTP/Emissions/Off Model Calculators/helper/calcs.py

@@ -0,0 +1,117 @@
+'''
+Class for the common calculator methods.
+'''
+import shutil
+import pandas as pd
+import re
+
+from helper import (mons,runs)
+
+
+class Calc:
+
+    def __init__(self, args, verbose=False):
+        self.runs = [args.model_run_id_2035, args.model_run_id_2050]
+        self.modelDataPath, self.masterFilePath = mons.get_directory_constants(args.d)
+        self.masterWbName=""
+        self.dataFileName=""
+        self.verbose=verbose
+        self.varsDir=mons.get_vars_directory(args.d)
+        self.v=Calc.get_variable_locations(self)
+
+    def copy_workbook(self):
+        # Start run
+        newWbFilePath=runs.createNewRun(self.runs)
+
+        # make a copy of the workbook
+        master_workbook_file = f"{self.masterFilePath}/{self.masterWbName}.xlsx"
+        self.new_workbook_file = f"{newWbFilePath}/{self.masterWbName}__{self.runs[0]}__{self.runs[1]}.xlsx"
+
+
+        shutil.copy2(master_workbook_file, self.new_workbook_file)
+
+        if self.verbose:
+            print(master_workbook_file)
+            print(self.new_workbook_file)
+
+        return self.new_workbook_file
+
+    def get_model_metadata(self):
+        metaData=pd.read_csv(
+            f"{self.modelDataPath}/{self.dataFileName}.csv",
+            nrows=0)
+
+        if self.verbose:
+            print(f"Model Data (R Script) metadata:\n{metaData.columns[0]}")
+
+        return metaData.columns[0]
+
+    def get_model_data(self):
+        # Get Model Data as df
+        rawData=pd.read_csv(
+            f"{self.modelDataPath}/{self.dataFileName}.csv",
+            skiprows=1)
+
+        filteredData=rawData.loc[rawData.directory.isin(self.runs)]
+
+        # Get metadata from model data
+        metaData=Calc.get_model_metadata(self)
+
+        if self.verbose:
+            print("Unique directories:")
+            print(rawData['directory'].unique())
+
+        return filteredData, metaData
+
+    def get_ipa(self, arg):
+
+        pattern = r"(\d{4})_(TM\d{3})_(.*)"
+        matches = re.search(pattern, arg)
+
+        if matches:
+            ipa = matches.group(3)
+            year = matches.group(1)
+
+        if self.verbose:
+            print(ipa)
+
+        return [ipa, int(year)]
+
+    def write_model_data_to_excel(self, data, meta):
+
+        with pd.ExcelWriter(self.new_workbook_file, engine='openpyxl', mode = 'a', if_sheet_exists = 'replace') as writer:  
+        # add metadata
+            meta=pd.DataFrame([meta])
+            meta.to_excel(writer, 
+                        sheet_name='Model Data', 
+                        index=False,
+                        header=False, 
+                        startrow=0, startcol=0)
+        with pd.ExcelWriter(self.new_workbook_file, engine='openpyxl', mode = 'a', if_sheet_exists = 'overlay') as writer:  
+        # add model data
+        # this only works with pandas=1.4.3 or later; in earlier version, it will not overwrite sheet, but add new one with sheet name 'Model Data1'
+            data.to_excel(writer, 
+                        sheet_name='Model Data', 
+                        index=False, 
+                        startrow=1, startcol=0)
+
+        if self.verbose:
+            print(f"Metadata: {meta}")
+
+    def get_variable_locations(self):
+
+        allVars=pd.read_excel(self.varsDir)
+        calcVars=allVars.loc[allVars.Workbook.isin([self.masterWbName])].drop(columns=['Workbook','Description'])
+        groups=set(calcVars.Sheet)
+        self.v={}
+        for group in groups:
+            self.v.setdefault(group,dict())
+            self.v[group]=dict(zip(calcVars['Variable Name'],calcVars['Location']))
+
+        if self.verbose:
+            print("Calculator variables and locations in Excel:")
+            print(self.v)
+
+
+
+

utilities/RTP/Emissions/Off Model Calculators/helper/mons.py

@@ -0,0 +1,75 @@
+'''
+This module includes common functions within the main script.
+'''
+
+def get_directory_constants(dirType):
+    '''
+    This function extracts the corresponding relative or absolute paths
+    used in the external or mtc options.
+    '''
+    # directory file paths (input, models, outputs)
+    if dirType=="external":
+        from templates.external import (
+                        MODEL_DATA_BOX_DIR,
+                        OFF_MODEL_CALCULATOR_DIR,
+                        )
+    else:
+        from templates.mtc import (
+                    MODEL_DATA_BOX_DIR,
+                    OFF_MODEL_CALCULATOR_DIR,
+                    )
+
+    return MODEL_DATA_BOX_DIR, OFF_MODEL_CALCULATOR_DIR
+
+def get_vars_directory(dirType):
+    # directory file paths (input, models, outputs)
+    if dirType=="external":
+        from templates.external import (
+                        VARS
+                        )
+    else:
+        from templates.mtc import (
+                    VARS
+                    )
+
+    return VARS
+
+# method depricated, now file names added inside subclasses
+def get_master_and_data_file_names(calcChoice):
+
+    if calcChoice=='bike_share':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_Bikeshare" 
+        MODEL_DATA_FILE_NAME="Model Data - Bikeshare"
+
+    elif calcChoice=='car_share':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_Carshare" 
+        MODEL_DATA_FILE_NAME="Model Data - Carshare"
+
+    elif calcChoice=='ebike':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_EBIKE" 
+        MODEL_DATA_FILE_NAME=None # Needs model data name
+
+    elif calcChoice=='regional_charger':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_RegionalCharger" 
+        MODEL_DATA_FILE_NAME=None # Needs model data name
+
+    elif calcChoice=='regional_charger_accii':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_RegionalCharger_ACCII" 
+        MODEL_DATA_FILE_NAME=None # Needs model data name
+
+    elif calcChoice=='targeted_trans_alt':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_TargetedTransAlt" 
+        MODEL_DATA_FILE_NAME="Model Data - Targeted Transportation Alternatives"
+
+    elif calcChoice=='vanpools':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_Vanpools" 
+        MODEL_DATA_FILE_NAME="Model Data - Employer Shuttles" # confirm  name
+
+    elif calcChoice=='vehicle_buyback':
+        MASTER_WORKBOOK_NAME="PBA50+_OffModel_VehicleBuyback" 
+        MODEL_DATA_FILE_NAME=None # Needs model data name
+
+    else:
+        raise ValueError("Choice not in options. Check the calculator name is correct.")
+
+    return MASTER_WORKBOOK_NAME, MODEL_DATA_FILE_NAME

utilities/RTP/Emissions/Off Model Calculators/helper/runs.py

@@ -0,0 +1,39 @@
+import os
+from templates.external import (
+    OFF_MODEL_CALCULATOR_DIR_OUTPUT
+    )
+
+def getNextFilePath(output_folder, run):
+
+    lastRunId=0
+    for f in os.listdir(output_folder):
+        fileNameList=f.split("__")
+        if f"{fileNameList[0]}__{fileNameList[1]}"== run:
+            if int(fileNameList[2])>lastRunId:
+                lastRunId=int(fileNameList[2])
+
+        else:
+            continue
+
+    return lastRunId + 1
+
+def createNewRun(model_run_ids_list, verbose=False):
+
+    runName=model_run_ids_list[0]+"__"+model_run_ids_list[0]
+    pathToRun=f"{OFF_MODEL_CALCULATOR_DIR_OUTPUT}/{runName}__0"
+
+    if not os.path.exists(pathToRun):
+        runNameNumber=runName+"__0"
+        os.makedirs(pathToRun)
+
+    else:
+        runID=getNextFilePath(OFF_MODEL_CALCULATOR_DIR_OUTPUT,runName)
+        runNameNumber=f"{runName}__{runID}"
+        pathToRun=f"{OFF_MODEL_CALCULATOR_DIR_OUTPUT}/{runNameNumber}"
+        os.makedirs(pathToRun)
+
+    if verbose:
+        print(f"New run created: {runNameNumber}")
+        print(f"Location: {pathToRun}")
+
+    return pathToRun

utilities/RTP/Emissions/Off Model Calculators/templates/external.py

@@ -0,0 +1,17 @@
+
+import os
+
+# from update_offmodel_calculator_workbooks_with_TM_output import ABS_DIRNAME
+# ABS_DIRNAME=r"C:\Users\63330\Documents\projects\MTC\travel-model-one\utilities\RTP\Emissions\Off Model Calculators".replace("\\","/")
+ABS_DIRNAME=os.path.join(os.path.dirname(__file__),"..").replace("\\","/")
+# Input data paths
+BOX_DIR = ABS_DIRNAME+r"\data\input\IPA_TM2".replace("\\","/")
+MODEL_DATA_BOX_DIR = BOX_DIR+"/ModelData"
+
+# Models
+OFF_MODEL_CALCULATOR_DIR = ABS_DIRNAME+r"\models".replace("\\","/")
+# Output
+OFF_MODEL_CALCULATOR_DIR_OUTPUT = ABS_DIRNAME+r"\data\output".replace("\\","/")
+
+# Variables locations
+VARS=ABS_DIRNAME+r"\models\Variable_locations.xlsx".replace("\\","/")

utilities/RTP/Emissions/Off Model Calculators/templates/mtc.py

@@ -0,0 +1,21 @@
+
+import os
+
+'''
+Import the absolute paths used within the MTC team. 
+E.g. links to box or other local directories.
+'''
+# Input data paths
+BOX_DIR = 'C:\Users\{}\Box\Plan Bay Area 2050+\Blueprint\Off-Model\PBA50+ Off-Model'\
+    .format(os.environ.get('USERNAME'))
+MODEL_DATA_BOX_DIR = os.path.join(BOX_DIR, 'model_data_all')
+
+# Models
+OFF_MODEL_CALCULATOR_DIR = os.path.join(
+    BOX_DIR, 'DBP_v2', 'PBA50+ Off-Model Calculators')
+
+# Outputs
+OFF_MODEL_CALCULATOR_DIR_OUTPUT = OFF_MODEL_CALCULATOR_DIR
+
+# Variables locations
+VARS=r"utilities\RTP\Emissions\Off Model Calculators\models\Variable_locations.xlsx".replace("\\","/")