Merge pull request #143 from CMAP-REPOS/c23q2

c23q2

.gitignore

@@ -1,5 +1,9 @@
+# Ignore temporary files generated by tools.
+temp/
+
 # Compiled source #
 ###################
+src/__pycache__/
 *.com
 *.class
 *.dll

README.md

@@ -1,7 +1,10 @@
-CMAP-REPOS/mhn_programs
-=======================
+# CMAP-REPOS/mhn_programs
+
 The MHN Programs repository is a collection of (mostly) Python scripts used to administer the [Chicago Metropolitan Agency for Planning (CMAP)](http://www.cmap.illinois.gov)'s Master Highway Network [geodatabase](http://www.esri.com/software/arcgis/geodatabase). This geodatabase is used, in conjunction with the Master Rail Network, to generate travel demand modeling networks, which we use for all of our modeling needs, including [transportation conformity](http://www.cmap.illinois.gov/conformity-analysis).
 
-The MHN itself contains information about all of the major roads within the 21-county CMAP modeling area, as well as all major road construction projects scheduled between now and 2040, all current CTA and Pace bus routes, and planned future bus routes (like [CTA's Ashland BRT](http://www.transitchicago.com/ashlandbrt)).
+The MHN itself contains information about all of the major roads within the 21-county CMAP modeling area, as well as all major road construction projects scheduled between now and 2050, all current CTA and Pace bus routes, and planned future bus routes.
 
 The scripts in this repository are used to import new GTFS bus data and road construction project details, maintain the integrity of the network after geometric edits have been made, and export data in a format suitable for input into [Emme](http://www.inrosoftware.com/en/products/emme) modeling networks. This repository includes an ArcGIS Toolbox (mhn_tools.tbx), which has been configured such that each of the main scripts can (and should!) be run within the ArcMap/ArcCatalog/ArcGIS Pro GUI out of the box!
+
+## Getting Started
+See the [wiki](https://github.com/CMAP-REPOS/mhn_programs/wiki/Getting-Set-Up).

MHN.py → src/MHN.py

@@ -289,16 +289,15 @@ def __init__(self, mhn_gdb_path, zone_gdb_path=None):
 
         # Directories
         self.root_dir = os.path.dirname(self.gdb)
-        self.imp_dir = self.ensure_dir(os.path.join(self.root_dir, 'import'))
-        self.out_dir = self.ensure_dir(os.path.join(self.root_dir, 'output'))
-        self.temp_dir = self.ensure_dir(os.path.join(self.root_dir, 'temp'))
         self.script_dir = sys.path[0]  # Directory containing this module
         if os.path.basename(self.script_dir) == 'utilities':
-            self.prog_dir = os.path.dirname(self.script_dir)
+            self.src_dir = os.path.dirname(self.script_dir)
             self.util_dir = self.script_dir
         else:
-            self.prog_dir = self.script_dir
-            self.util_dir = os.path.join(self.prog_dir, 'utilities')
+            self.src_dir = self.script_dir
+            self.util_dir = os.path.join(self.src_dir, 'utilities')
+        self.temp_dir = self.ensure_dir(os.path.realpath(os.path.join(self.src_dir, '../temp')))
+        self.in_dir = os.path.realpath(os.path.join(self.src_dir, '../input'))
         self.mem = 'in_memory'
 
         # MHN geodatabase structure, projection
@@ -688,7 +687,7 @@ def submit_sas(self, sas_file, sas_log, sas_lst, arg_list=None):
             arg_str = ''
         else:
             arg_str = '$'.join(str(arg) for arg in arg_list)
-        bat = os.path.join(self.prog_dir, 'sasrun.bat')
+        bat = os.path.join(self.src_dir, 'sasrun.bat')
         cmd = [bat, sas_file, arg_str, sas_log, sas_lst]
         # arcpy.AddMessage('{}: {}'.format(sas_file, arg_str))  # Helpful for debugging
         return subprocess.check_call(cmd, startupinfo=startupinfo)

generate_highway_files.py → src/generate_highway_files.py

@@ -122,7 +122,7 @@ def generate_node_zones_csv(out_csv):
 arcpy.Delete_management(overlap_network_view)
 
 # Process attribute tables with coding_overlap.sas.
-sas1_sas = ''.join((MHN.prog_dir, '/', sas1_name, '.sas'))
+sas1_sas = ''.join((MHN.src_dir, '/', sas1_name, '.sas'))
 sas1_args = [MHN.temp_dir]
 MHN.submit_sas(sas1_sas, sas1_log, sas1_lst, sas1_args)
 if not os.path.exists(sas1_log):
@@ -200,7 +200,7 @@ def generate_node_zones_csv(out_csv):
     arcpy.Delete_management(hwy_nodes_view)
 
     # Process attribute tables with generate_highway_files_2.sas.
-    sas2_sas = os.path.join(MHN.prog_dir, '{}.sas'.format(sas2_name))
+    sas2_sas = os.path.join(MHN.src_dir, '{}.sas'.format(sas2_name))
     sas2_args = [hwy_path, scen, MHN.max_poe, MHN.base_year, int(abm_output)]
     MHN.submit_sas(sas2_sas, sas2_log, sas2_lst, sas2_args)
     if not os.path.exists(sas2_log):

generate_transit_files.py → src/generate_transit_files.py

@@ -33,7 +33,6 @@
 MHN = MasterHighwayNetwork(mhn_gdb_path)
 scen_list = arcpy.GetParameterAsText(1).split(';')  # Semicolon-delimited string, e.g. '100;200'
 root_path = arcpy.GetParameterAsText(2)             # String, no default
-abm_output = arcpy.GetParameter(3)                  # Boolean, default = False
 
 out_tod_periods = sorted(MHN.tod_periods['transit'].keys())
 
@@ -64,9 +63,9 @@
 bus_itin_csv = os.path.join(MHN.temp_dir, 'bus_itin.csv')
 oneline_itin_txt = os.path.join(MHN.temp_dir, 'oneline_itin.txt')  # gtfs_collapse_routes.py input file (called by gtfs_reformat_feed.sas)
 feed_groups_txt = os.path.join(MHN.temp_dir, 'feed_groups.txt')    # gtfs_collapse_routes.py output file
-missing_links_csv = os.path.join(MHN.out_dir, 'missing_bus_links.csv')
-link_dict_txt = os.path.join(MHN.out_dir, 'link_dictionary.txt')  # shortest_path.py input file (called by generate_transit_files_2.sas)
-short_path_txt = os.path.join(MHN.out_dir, 'short_path.txt')      # shortest_path.py output file
+missing_links_csv = os.path.join(MHN.temp_dir, 'missing_bus_links.csv')
+link_dict_txt = os.path.join(MHN.temp_dir, 'link_dictionary.txt')  # shortest_path.py input file (called by generate_transit_files_2.sas)
+short_path_txt = os.path.join(MHN.temp_dir, 'short_path.txt')      # shortest_path.py output file
 path_errors_txt = os.path.join(MHN.temp_dir, 'path_errors.txt')
 
 
@@ -144,15 +143,15 @@
         arcpy.Delete_management(bus_itin_view)
 
         # Process exported route & itin tables with gtfs_reformat_feed.sas.
-        sas1_sas = os.path.join(MHN.prog_dir, '{}.sas'.format(sas1_name))
+        sas1_sas = os.path.join(MHN.src_dir, '{}.sas'.format(sas1_name))
         sas1_output = os.path.join(MHN.temp_dir, 'bus_{}_runs_{}.csv'.format(which_bus, tod))
-        sas1_args = [MHN.prog_dir, bus_route_csv, bus_itin_csv, oneline_itin_txt, feed_groups_txt, sas1_output, tod]
+        sas1_args = [MHN.src_dir, bus_route_csv, bus_itin_csv, oneline_itin_txt, feed_groups_txt, sas1_output, tod]
         MHN.delete_if_exists(sas1_output)
         MHN.submit_sas(sas1_sas, sas1_log, sas1_lst, sas1_args)
         if not os.path.exists(sas1_log):
             MHN.die('{} did not run!'.format(sas1_sas))
         elif not os.path.exists(feed_groups_txt):
-            MHN.die('{} did not run! (Called by {}.)'.format(os.path.join(MHN.prog_dir, 'gtfs_collapse_routes.py'), sas1_sas))
+            MHN.die('{} did not run! (Called by {}.)'.format(os.path.join(MHN.src_dir, 'gtfs_collapse_routes.py'), sas1_sas))
         elif os.path.exists(sas1_lst) or not os.path.exists(sas1_output):
             MHN.die('{} did not run successfully. Please review {}.'.format(sas1_sas, sas1_log))
         else:
@@ -293,10 +292,7 @@
         arcpy.Delete_management(bus_lyr)
 
         # Export header info of representative bus runs in current TOD.
-        if abm_output:
-            rep_runs_attr = [bus_id_field, 'DESCRIPTION', 'MODE', 'CT_VEH', 'SPEED', 'GROUP_HEADWAY']  # CT_VEH instead of VEHICLE_TYPE
-        else:
-            rep_runs_attr = [bus_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'SPEED', 'GROUP_HEADWAY']
+        rep_runs_attr = [bus_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'SPEED', 'GROUP_HEADWAY']
         rep_runs_query = MHN.tod_periods['transit'][tod][1]
         rep_runs_view = MHN.make_skinny_table_view(rep_runs_table, 'rep_runs_view', rep_runs_attr, rep_runs_query)
         rep_runs_csv = os.path.join(scen_tran_path, 'rep_runs.csv')
@@ -321,10 +317,7 @@
             bus_future_lyr = 'future_lyr'
             arcpy.MakeFeatureLayer_management(MHN.bus_future, bus_future_lyr)
             bus_future_id_field = MHN.route_systems[MHN.bus_future][1]
-            if abm_output:
-                bus_future_attr = [bus_future_id_field, 'DESCRIPTION', 'MODE', 'CT_VEH', 'SPEED', 'HEADWAY']  # CT_VEH instead of VEHICLE_TYPE
-            else:
-                bus_future_attr = [bus_future_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'SPEED', 'HEADWAY']
+            bus_future_attr = [bus_future_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'SPEED', 'HEADWAY']
             bus_future_query = ''' "SCENARIO" LIKE '%{}%' '''.format(scen[0])  # SCENARIO field contains first character of applicable scenario codes
             bus_future_view = MHN.make_skinny_table_view(bus_future_lyr, 'bus_future_view', bus_future_attr, bus_future_query)
             bus_future_csv = os.path.join(scen_tran_path, 'bus_future.csv')
@@ -650,11 +643,11 @@
         process_future = 1 if scen_year > MHN.base_year else 0
 
         # Call generate_transit_files_2.sas -- creates bus batchin files.
-        sas2_sas = os.path.join(MHN.prog_dir, '{}.sas'.format(sas2_name))
+        sas2_sas = os.path.join(MHN.src_dir, '{}.sas'.format(sas2_name))
         sas2_output = os.path.join(tran_path, '{}_{}.txt'.format(sas2_name, scen))
         sas2_args = (scen_tran_path, scen_hwy_path, rep_runs_csv, rep_runs_itin_csv, replace_csv, reroute_csv, pnr_csv,
                      scen, tod, str(min(MHN.centroid_ranges['CBD'])), str(max(MHN.centroid_ranges['CBD'])),
-                     str(MHN.max_poe), process_future, MHN.prog_dir, missing_links_csv,
+                     str(MHN.max_poe), process_future, MHN.src_dir, missing_links_csv,
                      link_dict_txt, short_path_txt, path_errors_txt, busway_links_csv, busway_nodes_csv,
                      sas2_output)
         if tod == out_tod_periods[0] and os.path.exists(sas2_output):
@@ -912,7 +905,7 @@ def distance_to_zone_centroid(pts_fc, pts_node_field, pts_zone_field, centroids_
             arcpy.Delete_management(fc)
 
         # Call generate_transit_files_3.sas -- writes access.network file.
-        sas3_sas = os.path.join(MHN.prog_dir, '{}.sas'.format(sas3_name))
+        sas3_sas = os.path.join(MHN.src_dir, '{}.sas'.format(sas3_name))
         sas3_output = os.path.join(scen_tran_path, 'access.network_{}'.format(tod))
         sas3_args = [scen_tran_path, scen, str(min(MHN.centroid_ranges['CBD'])), str(max(MHN.centroid_ranges['CBD'])), tod]
         MHN.submit_sas(sas3_sas, sas3_log, sas3_lst, sas3_args)
@@ -1055,7 +1048,7 @@ def get_scen_line_ids():
         w.write('{},1.0\n'.format(line_id))
 
 # Node extra attribute CSVs
-exec(open(os.path.join(MHN.prog_dir, 'transit_node_extra_attributes.py')).read())
+exec(open(os.path.join(MHN.src_dir, 'transit_node_extra_attributes.py')).read())
 
 # -----------------------------------------------------------------------------
 #  Clean up script-level data.

generate_transit_files_2.sas → src/generate_transit_files_2.sas

@@ -22,7 +22,7 @@ options noxwait;
 %let zone2 = %scan(&sysparm, 11, $);     * zone09 CBD end zone;
 %let maxzone = %scan(&sysparm, 12, $);   * highest zone09 POE zone number;
 %let procfut = %scan(&sysparm, 13, $);   * is scenario year later than MHN base year?;
-%let progdir = %scan(&sysparm, 14, $);
+%let srcdir = %scan(&sysparm, 14, $);
 %let misslink = %scan(&sysparm, 15, $);
 %let linkdict = %scan(&sysparm, 16, $);
 %let shrt = %scan(&sysparm, 17, $);
@@ -31,7 +31,7 @@ options noxwait;
 %let mode4nd = %scan(&sysparm, 20, $);
 %let outtxt = %scan(&sysparm, 21, $);
 %let shrtpath = %sysfunc(tranwrd(&shrt, /, \));
-%let pypath = %sysfunc(tranwrd(&progdir./pypath.txt, /, \));
+%let pypath = %sysfunc(tranwrd(&srcdir./pypath.txt, /, \));
 %let newln = 0;
 %let modln = 0;
 %let moditin = 0;
@@ -621,7 +621,7 @@ data _null_; set hold nobs=totobs;
 
             data _null_;
                 %put a=&a b=&b;
-                x "%str(%'&runpython.%') &progdir.\shortest_path.py &a &b &linkdict &shrtpath";
+                x "%str(%'&runpython.%') &srcdir.\shortest_path.py &a &b &linkdict &shrtpath";
             %let count = %eval(&count + 1);
         %end;
 

gtfs_reformat_feed.sas → src/gtfs_reformat_feed.sas

@@ -8,14 +8,14 @@
 */
 options noxwait;
 
-%let progdir = %scan(&sysparm, 1, $);
+%let srcdir = %scan(&sysparm, 1, $);
 %let busrte = %scan(&sysparm, 2, $);   * Bus header CSV;
 %let busitin = %scan(&sysparm, 3, $);  * Bus itinerary CSV;
 %let oneline = %scan(&sysparm, 4, $);  * One-line itineraries, passed to gtfs_collapse_routes.py;
 %let feedgrp = %scan(&sysparm, 5, $);  * Grouped bus routes, passed back from gtfs_collapse_routes.py;
 %let runs = %scan(&sysparm, 6, $);     * Final output CSV of this program;
 %let tod = %scan(&sysparm, 7, $);      * TOD period;
-%let pypath = %sysfunc(tranwrd(&progdir./pypath.txt, /, \));
+%let pypath = %sysfunc(tranwrd(&srcdir./pypath.txt, /, \));
 
 *~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*;
 filename in1 "&busitin";
@@ -71,7 +71,7 @@ data null; infile "&pypath" length=reclen obs=1;
     call symput('runpython', trim(location));
     run;
 
-x "%str(%'&runpython.%') &progdir.\gtfs_collapse_routes.py &oneline &feedgrp";
+x "%str(%'&runpython.%') &srcdir.\gtfs_collapse_routes.py &oneline &feedgrp";
 x "if exist &pypath (del &pypath /Q)"; run;
 
 

import_future_bus_routes.py → src/import_future_bus_routes.py

@@ -95,7 +95,7 @@
 # -----------------------------------------------------------------------------
 arcpy.AddMessage('{0}Validating coding in {1}...'.format('\n', xls))
 
-sas1_sas = os.path.join(MHN.prog_dir, '{0}.sas'.format(sas1_name))
+sas1_sas = os.path.join(MHN.src_dir, '{0}.sas'.format(sas1_name))
 sas1_args = [xls, MHN.temp_dir, future_itin_csv, future_route_csv, str(MHN.max_poe), sas1_lst]
 MHN.submit_sas(sas1_sas, sas1_log, sas1_lst, sas1_args)
 if not os.path.exists(sas1_log):
@@ -175,7 +175,7 @@
 
 route_fields = (
     common_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'HEADWAY', 'SPEED',
-    'SCENARIO', 'REPLACE', 'REROUTE', 'TOD', 'NOTES', 'CT_VEH'
+    'SCENARIO', 'REPLACE', 'REROUTE', 'TOD', 'NOTES'
 )
 
 with open(future_route_csv, 'r') as raw_routes:

import_future_bus_routes_2.sas → src/import_future_bus_routes_2.sas

@@ -80,7 +80,7 @@ data chk; set rte(where=(scenario in (.,0))); proc print; title "Scenario Values
 
 
     ** Replace File for ARC **;
-data rte; set rte (keep=tr_line des mode veh_type headway speed scenario replace reroute tod nt ct_veh);
+data rte; set rte (keep=tr_line des mode veh_type headway speed scenario replace reroute tod nt);
   label tr_line='TRANSIT_LINE'
         des='DESCRIPTION'
         mode='MODE'
@@ -91,8 +91,7 @@ data rte; set rte (keep=tr_line des mode veh_type headway speed scenario replace
         replace='REPLACE'
         reroute='REROUTE'
         tod='TOD'
-        nt='NOTES'
-        ct_veh='CT_VEH';
+        nt='NOTES';
    proc sort; by tr_line;
    proc export outfile=out1 dbms=csv label replace;
 

import_gtfs_bus_routes.py → src/import_gtfs_bus_routes.py

@@ -37,12 +37,12 @@
 nodes_csv = os.path.join(MHN.temp_dir, 'nodes.csv')
 header_csv = os.path.join(MHN.temp_dir, 'header.csv')
 itin_csv = os.path.join(MHN.temp_dir, 'itin.csv')
-link_dict_txt = os.path.join(MHN.out_dir, 'link_dictionary.txt')  # shortest_path.py input file (called by import_gtfs_bus_routes_2.sas)
-short_path_txt = os.path.join(MHN.out_dir, 'short_path.txt')      # shortest_path.py output file
-path_err_txt = os.path.join(MHN.out_dir, 'path_errors.txt')
-hold_check_csv = os.path.join(MHN.out_dir, 'hold_check.csv')
-hold_times_csv = os.path.join(MHN.out_dir, 'hold_times.csv')
-routes_processed_csv = os.path.join(MHN.out_dir, 'routes_processed.csv')
+link_dict_txt = os.path.join(MHN.temp_dir, 'link_dictionary.txt')  # shortest_path.py input file (called by import_gtfs_bus_routes_2.sas)
+short_path_txt = os.path.join(MHN.temp_dir, 'short_path.txt')      # shortest_path.py output file
+path_err_txt = os.path.join(MHN.temp_dir, 'path_errors.txt')
+hold_check_csv = os.path.join(MHN.temp_dir, 'hold_check.csv')
+hold_times_csv = os.path.join(MHN.temp_dir, 'hold_times.csv')
+routes_processed_csv = os.path.join(MHN.temp_dir, 'routes_processed.csv')
 
 
 # -----------------------------------------------------------------------------
@@ -130,10 +130,10 @@
 # -----------------------------------------------------------------------------
 arcpy.AddMessage('{0}Validating coding in {1} & {2}...'.format('\n', raw_header_csv, raw_itin_csv))
 
-sas1_sas = os.path.join(MHN.prog_dir, '{0}.sas'.format(sas1_name))
+sas1_sas = os.path.join(MHN.src_dir, '{0}.sas'.format(sas1_name))
 sas1_args = [
     raw_header_csv, raw_itin_csv, transact_csv, network_csv, nodes_csv,
-    MHN.prog_dir, header_csv, itin_csv, pseudo_csv, link_dict_txt,
+    MHN.src_dir, header_csv, itin_csv, pseudo_csv, link_dict_txt,
     short_path_txt, path_err_txt, hold_check_csv, hold_times_csv,
     routes_processed_csv, str(min_route_id), str(MHN.max_poe), sas1_lst
 ]
@@ -223,7 +223,7 @@
 
 route_fields = (
     common_id_field, 'DESCRIPTION', 'MODE', 'VEHICLE_TYPE', 'HEADWAY', 'SPEED',
-    'FEEDLINE', 'DIRECTION', 'CT_VEH', 'ROUTE_ID', 'START', 'AM_SHARE',
+    'FEEDLINE', 'DIRECTION', 'ROUTE_ID', 'START', 'AM_SHARE',
     'STARTHOUR', 'LONGNAME', 'TERMINAL'
 )
 

import_gtfs_bus_routes_2.sas → src/import_gtfs_bus_routes_2.sas

@@ -51,7 +51,7 @@ options noxwait;
 %let transact = %scan(&sysparm, 3, $);
 %let network = %scan(&sysparm, 4, $);
 %let nodes = %scan(&sysparm, 5, $);
-%let progdir = %scan(&sysparm, 6, $);
+%let srcdir = %scan(&sysparm, 6, $);
 %let head = %scan(&sysparm, 7, $);
 %let itin = %scan(&sysparm, 8, $);
 %let pseudo = %scan(&sysparm, 9, $);
@@ -64,7 +64,7 @@ options noxwait;
 %let counter = %scan(&sysparm, 16, $);
 %let maxzn = %scan(&sysparm, 17, $);
 %let lst = %scan(&sysparm, 18, $);
-%let pypath = %sysfunc(tranwrd(&progdir./pypath.txt, /, \));
+%let pypath = %sysfunc(tranwrd(&srcdir./pypath.txt, /, \));
 %let count = 1;
 %let tothold = 0;
 %let samenode = 0;
@@ -643,7 +643,7 @@ data temp; set hold nobs=totobs; call symput('tothold', left(put(totobs, 8.)));
 
             data _null_;
                 %put a=&a b=&b;
-                x "%str(%'&runpython.%') &progdir.\shortest_path.py &a &b &linkdict &shrtpath";
+                x "%str(%'&runpython.%') &srcdir.\shortest_path.py &a &b &linkdict &shrtpath";
             %let count = %eval(&count + 1);
         %end;
 
@@ -660,7 +660,7 @@ data temp; set hold nobs=totobs; call symput('tothold', left(put(totobs, 8.)));
             %end;
         /* ADD LOGIC TO VERIFY NUMBER OF ENTRIES IN SHORT_PATH IS EQUAL TO &TOTFIX, ELSE STOP SCRIPTS */
 
-        %include "&progdir./read_path_output.sas";
+        %include "&srcdir./read_path_output.sas";
     %end;
 %mend itinfix;
 %itinfix
@@ -742,8 +742,7 @@ data rte; set rte (keep=newline descr mode type headway speed line route_id rln
         trmnl='TERMINAL'
         start='START'
         strthour='STARTHOUR'
-        ampct='AM_SHARE'
-        vehicle='CT_VEH';
+        ampct='AM_SHARE';
    proc sort; by newline;
    proc export outfile=out1 dbms=csv label replace;
 

import_highway_projects.py → src/import_highway_projects.py

@@ -49,7 +49,7 @@
 mhn_links_view = MHN.make_skinny_table_view(MHN.arc, 'mhn_links_view', mhn_links_attr, mhn_links_query)
 MHN.write_attribute_csv(mhn_links_view, mhn_links_csv, mhn_links_attr)
 
-sas1_sas = os.path.join(MHN.prog_dir, '{0}.sas'.format(sas1_name))
+sas1_sas = os.path.join(MHN.src_dir, '{0}.sas'.format(sas1_name))
 sas1_args = [xls, mhn_links_csv, projects_csv, sas1_lst]
 MHN.submit_sas(sas1_sas, sas1_log, sas1_lst, sas1_args)
 if not os.path.exists(sas1_log):