Merge pull request #37 from VolpeUSDOT/FTOT_2022_1

Ftot 2022 1

changelog.md

@@ -1,5 +1,16 @@
 # FTOT Change Log
 
+## v2022_1
+The 2022.1 release provides updates related to emissions reporting, build costs for processors, and network density reduction (NDR). Supplementary FTOT-related resources and tools have also been expanded or updated. The following changes have been made:
+- Expanded emissions report: Default emission factors for non-CO2 pollutants (CO, CO2e, CH4, N2O, PM10, PM2.5, and VOC) have been added for rail and water modes. Previously default emission factors were provided for just road. See Section 5.7 for more information.
+- Commodity density input file: Users can include a new input file with a list of commodity densities to be used in emissions calculation. The pre-existing density conversion factor in the XML will be used as the default value in the absence of commodity-specific density information. See Section 5.2 for more information.
+- Processor build costs: The user can now specify pre-defined candidate processors with unique build costs, in addition to or instead of FTOT-generated candidates. These pre-defined candidates are input as if they were existing processor facilities, but for the inclusion of a build cost that is added to the total scenario if they are used at all.
+- Network Density Reduction (NDR) bug fixes: FTOT now calls the update_ndr_parameter() method in FTOT steps O, P, and D in addition to the original call in the G step to avoid a case where NDR is programmatically turned off in one step but not the others. Also, a SQL filter was added to prevent FTOT from trying to calculate the shortest path for an origin-destination pair that does not exist in the subgraph of accepted modes for a commodity.
+- Changes to the scenario XML schema and templates: The scenario XML schema, template file, and Quick Start XMLs have been modified with (1) the addition of an optional element for a commodity densities input file and (2) updated default CO2 emission factors for rail and water modes.
+- FTOT setup script: To ensure successful installation of FTOT’s Python dependencies, the file simple_setup.bat now specifies version 2.9.0 for the imageio package.
+- New Resources on FTOT-Public Wiki: The “Guidance on Creating New Scenarios” Wiki page summarizes important steps and helpful tools for creating and updating FTOT scenarios. The “Adding Segments to the FTOT Network” Wiki page walks FTOT users through the process of adding segments to customize the existing FTOT Multimodal network. As part of this update, Section 3.2 of the main documentation was also updated to provide further information on the default FTOT network.
+- Link Removal Tool: The Network Resiliency and Link Removal Tool is a separate tool that assesses network resilience by ranking and sequentially removing links in the optimal solution and rerunning scenarios in FTOT to evaluate the resilience of optimal solution costs in the face of disruption. The Link Removal tool has been updated to Python 3 and is now based on FTOT 2021.4. The original version of the tool was based on FTOT 2020.3.
+
 ## v2021_4
 The 2021.4 release provides updates related to emissions reporting, transport costs, as well as enhancements to output files and runtime. The following changes have been made:
 - Expanded emissions report: Users can now generate a separate emissions report with total emissions by commodity and mode for seven non-CO2 pollutants (CO, CO2e, CH4, N2O, PM10, PM2.5, and VOC). For this feature, FTOT includes a set of default non-CO2 emission factors for road which users can update. The CO2 emissions factor is still reported in the main FTOT outputs and the emissions factor can be adjusted in the XML.

program/ftot.py

@@ -26,9 +26,9 @@
     ureg.define('us_ton = US_ton')
 
 
-FTOT_VERSION = "2021.4"
-SCHEMA_VERSION = "6.0.1"
-VERSION_DATE = "1/10/2022"
+FTOT_VERSION = "2022.1"
+SCHEMA_VERSION = "6.0.2"
+VERSION_DATE = "4/1/2022"
 
 # ===================================================================================================
 

program/ftot_facilities.py

@@ -81,7 +81,8 @@ def db_cleanup_tables(the_scenario, logger):
         main_db_con.execute("drop table if exists facilities;")
         logger.debug("create the facilities table")
         main_db_con.executescript(
-            "create table facilities(facility_ID INTEGER PRIMARY KEY, location_id integer, facility_name text, facility_type_id integer, ignore_facility text, candidate binary, schedule_id integer, max_capacity float);")
+            """create table facilities(facility_ID INTEGER PRIMARY KEY, location_id integer, facility_name text, facility_type_id integer, 
+            ignore_facility text, candidate binary, schedule_id integer, max_capacity float, build_cost float);""")
 
         # facility_type_id table
         logger.debug("drop the facility_type_id table")
@@ -415,6 +416,14 @@ def check_for_input_error(input_type, input_val, filename, index, units=None):
                             "The entry is empty or non-numeric (check for extraneous characters)." \
                             .format(index, filename)
 
+    elif input_type == 'build_cost':
+        try:
+            float(input_val)
+        except ValueError:
+            error_message = "There is an error in the build_cost entry in row {} of {}. " \
+                            "The entry is empty or non-numeric (check for extraneous characters)." \
+                            .format(index, filename)
+
     return error_message
 
 
@@ -496,6 +505,26 @@ def load_facility_commodities_input_data(the_scenario, commodity_input_file, log
                 share_max_transport_distance = row["share_max_transport_distance"]
             else:
                 share_max_transport_distance = 'N'
+
+            # set to 0 if blank, otherwise convert to numerical after checkign for extra characters
+            if "build_cost" in list(row.keys()):
+                build_cost = row["build_cost"]
+                if build_cost == '':
+                    build_cost = 0
+                else:
+                    error_message = check_for_input_error("build_cost", build_cost, 
+                        commodity_input_file, index, units=commodity_unit)
+                    if error_message:
+                        raise Exception(error_message)
+                    else:
+                        build_cost = float(build_cost)
+            else:
+                build_cost = 0   
+
+            if build_cost > 0:
+                candidate_flag = 1
+            else:
+                candidate_flag = 0
 
             # add schedule_id, if available
             if "schedule" in list(row.keys()):
@@ -540,7 +569,7 @@ def load_facility_commodities_input_data(the_scenario, commodity_input_file, log
             temp_facility_commodities_dict[facility_name].append([facility_type, commodity_name, commodity_quantity,
                                                                   commodity_unit, commodity_phase,
                                                                   commodity_max_transport_distance, io,
-                                                                  share_max_transport_distance, max_processor_input,
+                                                                  share_max_transport_distance, candidate_flag, build_cost, max_processor_input,
                                                                   schedule_name])
 
     logger.debug("finished: load_facility_commodities_input_data")
@@ -560,15 +589,17 @@ def populate_facility_commodities_table(the_scenario, commodity_input_file, logg
 
     facility_commodities_dict = load_facility_commodities_input_data(the_scenario, commodity_input_file, logger)
 
+    #recognize generated candidate processors
     candidate = 0
+    generated_candidate = 0
     if os.path.split(commodity_input_file)[1].find("ftot_generated_processor_candidates") > -1:
-        candidate = 1
+        generated_candidate = 1
 
     # connect to main.db and add values to table
     # ---------------------------------------------------------
     with sqlite3.connect(the_scenario.main_db) as db_con:
         for facility_name, facility_data in iteritems(facility_commodities_dict):
-
+            
             # unpack the facility_type (should be the same for all entries)
             facility_type = facility_data[0][0]
             facility_type_id = get_facility_id_type(the_scenario, db_con, facility_type, logger)
@@ -581,17 +612,26 @@ def populate_facility_commodities_table(the_scenario, commodity_input_file, logg
 
             max_processor_input = facility_data[0][-2]
 
+            build_cost = facility_data[0][-3]
+
+            #recognize candidate processors from proc.csv or generated file
+            candidate_flag = facility_data[0][-4]
+            if candidate_flag == 1 or generated_candidate == 1:
+                candidate = 1
+            else:
+                candidate = 0
+
             # get the facility_id from the db (add the facility if it doesn't exists)
             # and set up entry in facility_id table
-            facility_id = get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger)
+            facility_id = get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, build_cost, logger)
 
             # iterate through each commodity
             for commodity_data in facility_data:
 
                 # get commodity_id. (adds commodity if it doesn't exist)
                 commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger)
 
-                [facility_type, commodity_name, commodity_quantity, commodity_units, commodity_phase, commodity_max_transport_distance, io, share_max_transport_distance, unused_var_max_processor_input, schedule_id] = commodity_data
+                [facility_type, commodity_name, commodity_quantity, commodity_units, commodity_phase, commodity_max_transport_distance, io, share_max_transport_distance, candidate_data, build_cost, unused_var_max_processor_input, schedule_id] = commodity_data
 
                 if not commodity_quantity == "0.0":  # skip anything with no material
                     sql = "insert into facility_commodities " \
@@ -763,12 +803,19 @@ def get_facility_location_id(the_scenario, db_con, facility_name, logger):
 # =============================================================================
 
 
-def get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger):
+def get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, build_cost, logger):
 
     #  if it doesn't exist, add to facilities table and generate a facility id.
-    db_con.execute("insert or ignore into facilities "
-                   "(location_id, facility_name, facility_type_id, candidate, schedule_id, max_capacity) "
-                   "values ('{}', '{}', {}, {}, {}, {});".format(location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input))
+    if build_cost > 0:
+        # specify ignore_facility = 'false'. Otherwise, the input-from-file candidates get ignored like excess generated candidates
+        ignore_facility = 'false'
+        db_con.execute("insert or ignore into facilities "
+                   "(location_id, facility_name, facility_type_id, ignore_facility, candidate, schedule_id, max_capacity, build_cost) "
+                   "values ('{}', '{}', {}, '{}',{}, {}, {}, {});".format(location_id, facility_name, facility_type_id, ignore_facility, candidate, schedule_id, max_processor_input, build_cost))
+    else:
+        db_con.execute("insert or ignore into facilities "
+                   "(location_id, facility_name, facility_type_id, candidate, schedule_id, max_capacity, build_cost) "
+                   "values ('{}', '{}', {},  {}, {}, {}, {});".format(location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, build_cost))
 
     # get facility_id
     db_cur = db_con.execute("select facility_id "
@@ -816,7 +863,7 @@ def get_facility_id_type(the_scenario, db_con, facility_type, logger):
 def get_commodity_id(the_scenario, db_con, commodity_data, logger):
 
     [facility_type, commodity_name, commodity_quantity, commodity_unit, commodity_phase, 
-     commodity_max_transport_distance, io, share_max_transport_distance, max_processor_input, schedule_id] = commodity_data
+     commodity_max_transport_distance, io, share_max_transport_distance, candidate, build_cost, max_processor_input, schedule_id] = commodity_data
 
     # get the commodity_id.
     db_cur = db_con.execute("select commodity_id "

program/ftot_networkx.py

@@ -528,9 +528,11 @@ def make_od_pairs(the_scenario, logger):
         '''
         db_cur.execute(sql)
 
+        sql = "drop table if exists tmp_od_pairs;"
+        db_cur.execute(sql)
+
         sql = '''
-        insert into od_pairs (from_location_id, to_location_id, from_facility_id, to_facility_id, commodity_id, 
-        phase_of_matter, from_node_id, to_node_id, from_location_1, to_location_1)
+        create table tmp_od_pairs as
         select distinct
         origin.location_id AS from_location_id,
         destination.location_id AS to_location_id,
@@ -584,9 +586,44 @@ def make_od_pairs(the_scenario, logger):
         '''
         db_cur.execute(sql)
 
+        sql = '''
+        insert into od_pairs (from_location_id, to_location_id, from_facility_id, to_facility_id, commodity_id, 
+        phase_of_matter, from_node_id, to_node_id, from_location_1, to_location_1)
+        select distinct
+        odp.from_location_id,
+        odp.to_location_id,
+        odp.from_facility_id,
+        odp.to_facility_id,
+        odp.commodity_id,
+        odp.phase_of_matter,
+        odp.from_node_id,
+        odp.to_node_id,
+        odp.from_location_1,
+        odp.to_location_1
+        from tmp_od_pairs odp
+        inner join networkx_edges ne1 on
+        odp.from_node_id = ne1.from_node_id
+        inner join networkx_edges ne2 on
+        odp.to_node_id = ne2.to_node_id
+        inner join (select commodity_id, mode
+        from commodity_mode
+        where allowed_yn = 'Y') cm1 on
+        odp.commodity_id = cm1.commodity_id
+        and ne1.mode_source = cm1.mode
+        inner join (select commodity_id, mode
+        from commodity_mode
+        where allowed_yn = 'Y') cm2 on
+        odp.commodity_id = cm2.commodity_id
+        and ne2.mode_source = cm2.mode;
+        '''
+        db_cur.execute(sql)
+
         logger.debug("drop the tmp_connected_facilities_with_commodities table")
         db_cur.execute("drop table if exists tmp_connected_facilities_with_commodities;")
 
+        logger.debug("drop the tmp_od_pairs table")
+        db_cur.execute("drop table if exists tmp_od_pairs;")
+
         logger.info("end: create o-d pairs table")
 
         # Fetch all od-pairs, ordered by target

program/ftot_postprocess.py

@@ -26,6 +26,11 @@ def route_post_optimization_db(the_scenario, logger):
     parsed_optimal_solution = parse_optimal_solution_db(the_scenario, logger)
     optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material = parsed_optimal_solution
 
+    from ftot_networkx import update_ndr_parameter
+
+    # Check NDR conditions before post-processing results
+    update_ndr_parameter(the_scenario, logger)
+
     if not the_scenario.ndrOn:
         # Make the optimal routes and route_segments FCs from the db
         # ------------------------------------------------------------
@@ -1329,7 +1334,8 @@ def detailed_emissions_setup(the_scenario, logger):
             constraint unique_elements unique(commodity, mode, measure))
             ;""")
 
-        attributes_dict = ftot_supporting_gis.get_commodity_vehicle_attributes_dict(the_scenario, logger, EmissionsWarning=True)
+        # set isEmissionsReporting to True to suppress duplicate logger statements when calling ftot_supporting_gis for the second time
+        attributes_dict = ftot_supporting_gis.get_commodity_vehicle_attributes_dict(the_scenario, logger, isEmissionsReporting=True)
 
         sql_mode_commodity = "select network_source_id, commodity_name from optimal_route_segments group by network_source_id, commodity_name;"
         db_cur = db_con.execute(sql_mode_commodity)

program/ftot_processor.py

@@ -129,8 +129,10 @@ def populate_candidate_process_commodities(the_scenario, candidate_process_commo
             io = commodity[6]
             shared_max_transport_distance = 'N'
             processor_max_input = commodity[3]
+            build_cost = 0
+            candidate = 1
             # empty string for facility type and schedule id because fields are not used
-            commodity_data = ['', commodity_name, commodity_quantity, commodity_unit, commodity_phase, commodity_max_transport_dist, io, shared_max_transport_distance, processor_max_input, '']
+            commodity_data = ['', commodity_name, commodity_quantity, commodity_unit, commodity_phase, commodity_max_transport_dist, io, shared_max_transport_distance, candidate, build_cost, processor_max_input, '']
 
             # get commodity_id. (adds commodity if it doesn't exist)
             commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger)