From 1118a1cb5d306ae05b2f3d005ca71039706d560c Mon Sep 17 00:00:00 2001 From: kzhang81 Date: Mon, 10 Jan 2022 21:24:03 -0500 Subject: [PATCH] Pulling over 2021.4 changes from FTOT Volpe repository. --- .gitattributes | 42 +- .gitignore | 26 +- LICENSE | 264 +- README.md | 83 +- changelog.md | 359 +- program/ftot.py | 629 +- program/ftot_facilities.py | 2416 +++---- program/ftot_maps.py | 2058 +++--- program/ftot_networkx.py | 2512 +++---- program/ftot_postprocess.py | 3805 +++++----- program/ftot_processor.py | 2378 +++---- program/ftot_pulp.py | 7348 ++++++++++---------- program/ftot_pulp_candidate_generation.py | 5173 +++++++------- program/ftot_pulp_sourcing.py | 4259 ++++++------ program/ftot_report.py | 1118 +-- program/ftot_routing.py | 2342 +++---- program/ftot_scenario.py | 1345 ++-- program/ftot_setup.py | 478 +- program/ftot_supporting.py | 1274 ++-- program/ftot_supporting_gis.py | 1321 ++-- program/lib/Master_FTOT_Schema.xsd | 791 +-- program/lib/detailed_emission_factors.csv | 65 + program/lib/v6_temp_Scenario.xml | 346 +- program/lib/vehicle_types.csv | 16 +- program/tools/facility_data_tools.py | 248 +- program/tools/ftot_tools.py | 202 +- program/tools/gridded_data_tool.py | 224 +- program/tools/input_csv_templates_tool.py | 100 +- program/tools/lxml_upgrade_tool.py | 692 +- program/tools/network_disruption_tool.py | 542 +- program/tools/odpair_util2.py | 236 +- program/tools/odpairs_utils.py | 142 +- program/tools/run_upgrade_tool.py | 410 +- program/tools/scenario_compare_tool.py | 600 +- program/tools/xml_text_replacement_tool.py | 466 +- simple_setup.bat | 70 +- 36 files changed, 22400 insertions(+), 21980 deletions(-) create mode 100644 program/lib/detailed_emission_factors.csv diff --git a/.gitattributes b/.gitattributes index ab012ca..a9fc1dc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,22 +1,22 @@ -# Set the default behavior, in case people don't have core.autocrlf set. -* text eol=crlf - -#====================================================================================================================== -# EVERYTHING BELOW IS COMMENETED OUT. THE DEFAULT BEHAVIOR FOR THE FTOT -# REPOSITORY IS ALL TEXT EOL=CRLF AND THIS WILL OVERRIDE A USERS LOCAL -# core.autocrlf -# SEE THE DOCUMENTATION HERE: https://help.github.com/articles/dealing-with-line-endings/#per-repository-settings -#====================================================================================================================== - -# Explicitly declare text files you want to always be normalized and converted -# to native line endings on checkout. -#*.c text -#*.h text - -# Declare files that will always have CRLF line endings on checkout. -#*.sln text eol=crlf - -# Denote all files that are truly binary and should not be modified. -#*.png binary -#*.jpg binary +# Set the default behavior, in case people don't have core.autocrlf set. +* text eol=crlf + +#====================================================================================================================== +# EVERYTHING BELOW IS COMMENETED OUT. THE DEFAULT BEHAVIOR FOR THE FTOT +# REPOSITORY IS ALL TEXT EOL=CRLF AND THIS WILL OVERRIDE A USERS LOCAL +# core.autocrlf +# SEE THE DOCUMENTATION HERE: https://help.github.com/articles/dealing-with-line-endings/#per-repository-settings +#====================================================================================================================== + +# Explicitly declare text files you want to always be normalized and converted +# to native line endings on checkout. +#*.c text +#*.h text + +# Declare files that will always have CRLF line endings on checkout. +#*.sln text eol=crlf + +# Denote all files that are truly binary and should not be modified. +#*.png binary +#*.jpg binary *.whl binary \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6049558..440c0d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,14 @@ -.idea -.vs -documentation/ -scenarios/ -logs/ -temp_networkx_shp_files/ -debug/ -Maps/ -Reports/ -*.pyc -*.db -*.gdb -Maps_Time_Commodity/ +.idea +.vs +documentation/ +scenarios/ +logs/ +temp_networkx_shp_files/ +debug/ +Maps/ +Reports/ +*.pyc +*.db +*.gdb +Maps_Time_Commodity/ python3_env/ \ No newline at end of file diff --git a/LICENSE b/LICENSE index 71fae99..d85eb7e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,132 +1,132 @@ -Volpe Center -End User License Agreement -For Freight and Fuel Transportation Optimization Tool (FTOT) -IMPORTANT- READ CAREFULLY: THIS END USER LICENSE AGREEMENT -(“AGREEMENT”) IS A LEGAL AGREEMENT BETWEEN YOU (IN YOUR CAPACITY -AS AN INDIVIDUAL AND AS AN AGENT FOR YOUR COMPANY, INSTITUTION OR -OTHER ENTITY) (COLLECTIVELY, “YOU” OR “LICENSEE”) AND THE U.S. -DEPARTMENT OF TRANSPORTATION, JOHN A. VOLPE NATIONAL -TRANSPORTATION SYSTEMS CENTER (“VOLPE CENTER”). DOWNLOADING, -INSTALLING, USING, OR COPYING OF THE SOFTWARE (AS DEFINED BELOW) -BY YOU OR BY A THIRD PARTY ON YOUR BEHALF INDICATES YOUR -AGREEMENT TO BE BOUND BY THE TERMS AND CONDITIONS OF THIS -AGREEMENT. IF YOU DO NOT AGREE TO THESE TERMS AND CONDITIONS, DO -NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE. - -Description of FTOT: -“Freight and Fuel Transportation Optimization Tool (FTOT)” is the software tool that the -Volpe Center created to facilitate freight and fuel commodity scenario exploration to -evaluate optimal routing, flows, costs, emissions, and fuel demand for supply chains and -transportation analyses. -1. LICENSE. The Volpe Center grants you, and you hereby accept, a non-exclusive, non- -transferable, royalty-free license to install and use the software, in executable code -format only, together with any associated media, printed materials, and electronic -documentation (if any) provided by the Volpe Center (collectively, the “Software” or -“FTOT”), subject to the following terms and conditions: - -a) You may use the Software solely for your own internal use. You agree that you will not -use the Software for any commercial purpose. If you desire to use the Software -in any purpose other than use as described in this paragraph, you must -seek written permission from the Volpe Center by contacting by email -addressed to: kristin.lewis@dot.gov. -b) The software may be used solely by the individual downloading the software who has -agreed to the terms of this license. You may not transfer a copy of the Software to any -third party, nor may you allow the Software to be accessed over a network or the internet -in a manner that would allow users to access the Software who are not employed by you, -without the prior written consent of the Volpe Center; -c) You may copy the Software solely to the extent reasonably necessary to exercise the -foregoing license, and for backup and archival purposes; provided however that (i) you -must reproduce all proprietary notices on any copies of the Software and you must not -remove or alter those notices; (ii) all copies of the Software shall be subject to the terms -of this Agreement; and (iii) you may not otherwise copy or allow copies of the Software -to be made; (iv) you may not use the Volpe Center, U.S. Department of Transportation, -or U.S. government name nor any adaptation thereof, nor the names of any their -employees in any advertising, promotional or sales literature, except that you as the -licensee shall give appropriate credits in professional journals and publications as -follows: “Volpe, The National Transportation Systems Center. 2019. Freight and Fuel -Transportation Optimization Tool (FTOT). Cambridge, MA.”; and -d) You may not modify, alter, or create derivative works of the Software in any manner. You -may not rent, lease, loan, sublicense, distribute or transfer the Software to any third -party. - -2. NO MAINTENANCE OR SUPPORT. The Volpe Center is under no obligation -whatsoever to: (i) provide maintenance or support for the Software; or (ii) to notify you -of bug fixes, patches or upgrades to the Software (if any). If, in its sole discretion, the -Volpe Center makes a Software bug fix, patch or upgrade available to you and the Volpe -Center does not separately enter into a written license agreement with you relating to -such bug fix, patch or upgrade, then it shall be deemed incorporated into the Software -and subject to this Agreement. -3. LICENSE FEE. The software is provided at no cost to the licensee. -4. U.S. GOVERNMENT RIGHTS. The Software was developed by the Volpe -Center. The U.S. Department of Transportation and U.S. -Government hereby reserve all rights, title and interest in and to the -Software which are not explicitly granted to you herein; and without limiting the -generality of the foregoing, you do not acquire any rights, express or implied, in the -Software, other than those specifically set forth in this Agreement. -5. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED “AS IS” -WITHOUT WARRANTY OF ANY KIND. THE VOLPE CENTER, THE UNITED -STATES, THE UNITED STATES DEPARTMENT OF TRANSPORTATION, AND -THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON- -INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY OR -RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS -OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE -SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO -NOT WARRANT THAT THE SOFTWARE WILL FUNCTION -UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL BE -CORRECTED. -6. LIMITATION OF LIABILITY. IN NO EVENT WILL THE VOLPE CENTER, U.S. -DEPARTMENT OF TRANSPORTATION, UNITED STATES BE LIABLE FOR ANY -INDIRECT, INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE -DAMAGES OF ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED -TO LOSS OF PROFITS OR LOSS OF DATA, FOR ANY REASON -WHATSOEVER, WHETHER SUCH LIABILITY IS ASSERTED ON THE BASIS -OF CONTRACT, TORT (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR -OTHERWISE, EVEN IF THE VOLPE CENTER HAS BEEN WARNED OF THE -POSSIBILITY OF SUCH LOSS OR DAMAGES. IN NO EVENT SHALL THE VOLPE -CENTER’S LIABILITY FOR DAMAGES ARISING FROM OR IN CONNECTION WITH -THIS AGREEMENT EXCEED THE AMOUNT PAID BY YOU FOR THE SOFTWARE. -7. INDEMNITY. You shall indemnify, defend, and hold harmless the Volpe -Center, the U.S. Department of Transportation, the U.S. -Government, the Software developers, the Software sponsors, and their agents, -officers, and employees, against any and all claims, suits, losses, damage, costs, -fees, and expenses arising out of or in connection with this Agreement. You shall -pay all costs incurred by the Volpe Center in enforcing this provision, including -reasonable attorney fees. -8. TERM AND TERMINATION. The license granted to you under this Agreement will -continue perpetually unless terminated by the Volpe Center in accordance with this -Agreement. If you breach any term of this Agreement, and fail to cure such breach -within thirty (30) days of the date of written notice, this Agreement shall immediately -terminate. Upon any such termination, you shall immediately cease using the Software -and destroy all copies of the Software. Termination shall not relieve you from your -obligations arising prior to such termination. Notwithstanding any provision of this -Agreement to the contrary, Sections 4 through 10 shall survive termination of this -Agreement. -9. EXPORT CONTROLS. You shall observe all applicable United States and foreign laws -and regulations (if any) with respect to the export, re-export, diversion or transfer of -the Software, related technical data and direct products thereof, including, without -limitation, the International Traffic in Arms Regulations (ITAR) and the Export -Administration Regulations. The export of any technology from the United States, -including without limitation the Software and related technical data, may require some -form of export control license from the U.S. Government and, pursuant to U.S. laws, and -failure to obtain any required export control license may result in criminal liability under -U.S. laws. - 10. GENERAL. This Agreement shall be governed by the laws of the United States. No -provision in either party's purchase orders, or in any other business forms employed by -either party will supersede the terms of this Agreement, and no modification or -amendment of this Agreement is binding, unless in writing signed by a duly authorized -representative of each party. This Agreement is binding upon and shall inure to the -benefit of the Volpe Center, its successors and assigns. This Agreement represents the -entire understanding of the parties, and supersedes all previous communications, -written or oral, relating to the subject of this Agreement. If you have questions -concerning FTOT including, but not limited to 1) seeking permission for other uses; 2) -your interest in collaborating with the Volpe Center; and 3) providing feedback regarding -the software, please contact the Volpe Center via the GitHub repository by submitting -an Issue on the site. -BY DOWNLOADING, INSTALLING OR USING THE SOFTWARE, YOU ARE -INDICATING YOUR ACCEPTANCE OF THE TERMS AND CONDITIONS HEREIN. - - - - +Volpe Center +End User License Agreement +For Freight and Fuel Transportation Optimization Tool (FTOT) +IMPORTANT- READ CAREFULLY: THIS END USER LICENSE AGREEMENT +(“AGREEMENT”) IS A LEGAL AGREEMENT BETWEEN YOU (IN YOUR CAPACITY +AS AN INDIVIDUAL AND AS AN AGENT FOR YOUR COMPANY, INSTITUTION OR +OTHER ENTITY) (COLLECTIVELY, “YOU” OR “LICENSEE”) AND THE U.S. +DEPARTMENT OF TRANSPORTATION, JOHN A. VOLPE NATIONAL +TRANSPORTATION SYSTEMS CENTER (“VOLPE CENTER”). DOWNLOADING, +INSTALLING, USING, OR COPYING OF THE SOFTWARE (AS DEFINED BELOW) +BY YOU OR BY A THIRD PARTY ON YOUR BEHALF INDICATES YOUR +AGREEMENT TO BE BOUND BY THE TERMS AND CONDITIONS OF THIS +AGREEMENT. IF YOU DO NOT AGREE TO THESE TERMS AND CONDITIONS, DO +NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE. + +Description of FTOT: +“Freight and Fuel Transportation Optimization Tool (FTOT)” is the software tool that the +Volpe Center created to facilitate freight and fuel commodity scenario exploration to +evaluate optimal routing, flows, costs, emissions, and fuel demand for supply chains and +transportation analyses. +1. LICENSE. The Volpe Center grants you, and you hereby accept, a non-exclusive, non- +transferable, royalty-free license to install and use the software, in executable code +format only, together with any associated media, printed materials, and electronic +documentation (if any) provided by the Volpe Center (collectively, the “Software” or +“FTOT”), subject to the following terms and conditions: + +a) You may use the Software solely for your own internal use. You agree that you will not +use the Software for any commercial purpose. If you desire to use the Software +in any purpose other than use as described in this paragraph, you must +seek written permission from the Volpe Center by contacting by email +addressed to: kristin.lewis@dot.gov. +b) The software may be used solely by the individual downloading the software who has +agreed to the terms of this license. You may not transfer a copy of the Software to any +third party, nor may you allow the Software to be accessed over a network or the internet +in a manner that would allow users to access the Software who are not employed by you, +without the prior written consent of the Volpe Center; +c) You may copy the Software solely to the extent reasonably necessary to exercise the +foregoing license, and for backup and archival purposes; provided however that (i) you +must reproduce all proprietary notices on any copies of the Software and you must not +remove or alter those notices; (ii) all copies of the Software shall be subject to the terms +of this Agreement; and (iii) you may not otherwise copy or allow copies of the Software +to be made; (iv) you may not use the Volpe Center, U.S. Department of Transportation, +or U.S. government name nor any adaptation thereof, nor the names of any their +employees in any advertising, promotional or sales literature, except that you as the +licensee shall give appropriate credits in professional journals and publications as +follows: “Volpe, The National Transportation Systems Center. 2019. Freight and Fuel +Transportation Optimization Tool (FTOT). Cambridge, MA.”; and +d) You may not modify, alter, or create derivative works of the Software in any manner. You +may not rent, lease, loan, sublicense, distribute or transfer the Software to any third +party. + +2. NO MAINTENANCE OR SUPPORT. The Volpe Center is under no obligation +whatsoever to: (i) provide maintenance or support for the Software; or (ii) to notify you +of bug fixes, patches or upgrades to the Software (if any). If, in its sole discretion, the +Volpe Center makes a Software bug fix, patch or upgrade available to you and the Volpe +Center does not separately enter into a written license agreement with you relating to +such bug fix, patch or upgrade, then it shall be deemed incorporated into the Software +and subject to this Agreement. +3. LICENSE FEE. The software is provided at no cost to the licensee. +4. U.S. GOVERNMENT RIGHTS. The Software was developed by the Volpe +Center. The U.S. Department of Transportation and U.S. +Government hereby reserve all rights, title and interest in and to the +Software which are not explicitly granted to you herein; and without limiting the +generality of the foregoing, you do not acquire any rights, express or implied, in the +Software, other than those specifically set forth in this Agreement. +5. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED “AS IS” +WITHOUT WARRANTY OF ANY KIND. THE VOLPE CENTER, THE UNITED +STATES, THE UNITED STATES DEPARTMENT OF TRANSPORTATION, AND +THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON- +INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY OR +RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS +OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE +SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO +NOT WARRANT THAT THE SOFTWARE WILL FUNCTION +UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL BE +CORRECTED. +6. LIMITATION OF LIABILITY. IN NO EVENT WILL THE VOLPE CENTER, U.S. +DEPARTMENT OF TRANSPORTATION, UNITED STATES BE LIABLE FOR ANY +INDIRECT, INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE +DAMAGES OF ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED +TO LOSS OF PROFITS OR LOSS OF DATA, FOR ANY REASON +WHATSOEVER, WHETHER SUCH LIABILITY IS ASSERTED ON THE BASIS +OF CONTRACT, TORT (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR +OTHERWISE, EVEN IF THE VOLPE CENTER HAS BEEN WARNED OF THE +POSSIBILITY OF SUCH LOSS OR DAMAGES. IN NO EVENT SHALL THE VOLPE +CENTER’S LIABILITY FOR DAMAGES ARISING FROM OR IN CONNECTION WITH +THIS AGREEMENT EXCEED THE AMOUNT PAID BY YOU FOR THE SOFTWARE. +7. INDEMNITY. You shall indemnify, defend, and hold harmless the Volpe +Center, the U.S. Department of Transportation, the U.S. +Government, the Software developers, the Software sponsors, and their agents, +officers, and employees, against any and all claims, suits, losses, damage, costs, +fees, and expenses arising out of or in connection with this Agreement. You shall +pay all costs incurred by the Volpe Center in enforcing this provision, including +reasonable attorney fees. +8. TERM AND TERMINATION. The license granted to you under this Agreement will +continue perpetually unless terminated by the Volpe Center in accordance with this +Agreement. If you breach any term of this Agreement, and fail to cure such breach +within thirty (30) days of the date of written notice, this Agreement shall immediately +terminate. Upon any such termination, you shall immediately cease using the Software +and destroy all copies of the Software. Termination shall not relieve you from your +obligations arising prior to such termination. Notwithstanding any provision of this +Agreement to the contrary, Sections 4 through 10 shall survive termination of this +Agreement. +9. EXPORT CONTROLS. You shall observe all applicable United States and foreign laws +and regulations (if any) with respect to the export, re-export, diversion or transfer of +the Software, related technical data and direct products thereof, including, without +limitation, the International Traffic in Arms Regulations (ITAR) and the Export +Administration Regulations. The export of any technology from the United States, +including without limitation the Software and related technical data, may require some +form of export control license from the U.S. Government and, pursuant to U.S. laws, and +failure to obtain any required export control license may result in criminal liability under +U.S. laws. + 10. GENERAL. This Agreement shall be governed by the laws of the United States. No +provision in either party's purchase orders, or in any other business forms employed by +either party will supersede the terms of this Agreement, and no modification or +amendment of this Agreement is binding, unless in writing signed by a duly authorized +representative of each party. This Agreement is binding upon and shall inure to the +benefit of the Volpe Center, its successors and assigns. This Agreement represents the +entire understanding of the parties, and supersedes all previous communications, +written or oral, relating to the subject of this Agreement. If you have questions +concerning FTOT including, but not limited to 1) seeking permission for other uses; 2) +your interest in collaborating with the Volpe Center; and 3) providing feedback regarding +the software, please contact the Volpe Center via the GitHub repository by submitting +an Issue on the site. +BY DOWNLOADING, INSTALLING OR USING THE SOFTWARE, YOU ARE +INDICATING YOUR ACCEPTANCE OF THE TERMS AND CONDITIONS HEREIN. + + + + diff --git a/README.md b/README.md index 77f77ec..edee584 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,41 @@ -# FTOT -Freight and Fuel Transportation Optimization Tool - -## Description: -FTOT is a flexible scenario-testing tool that optimizes the transportation of materials for future energy and freight -scenarios. FTOT models and tracks commodity-specific information and can take into account conversion of raw materials to products (e.g., crude oil to jet fuel and diesel) and the fulfillment of downstream demand. FTOT was developed at the US Dept. of Transportation's Volpe National Transportation Systems Center. - -## Installation: -See [FTOT Installation wiki](https://github.com/VolpeUSDOT/FTOT-Public/wiki/FTOT-Installation-Guide) for the latest detailed instructions. If you need to run a legacy copy of FTOT compatible with ArcGIS rather than ArcGIS Pro, consult these [separate legacy instructions](https://github.com/VolpeUSDOT/FTOT-Public/wiki/FTOT-Installation-Guide-(ArcGIS-Legacy)) -* FTOT is a python based tool. -* Clone or download the repository. -* Install the required dependencies (including ESRI ArcGIS Pro) -* Download the [documentation and scenario dataset](https://github.com/VolpeUSDOT/FTOT-Public/wiki/Documentation-and-Scenario-Datasets) - -## Usage: -* Usage is explained in the Quick Start documentation here: [Documentation and Scenario Dataset Wiki](https://github.com/VolpeUSDOT/FTOT-Public/wiki/Documentation-and-Scenario-Datasets) - -## Contributing: -* Add bugs and feature requests to the Issues tab for the Volpe Development Team to triage. - -## Credits: -* Dr. Kristin Lewis (Volpe) -* Matthew Pearlson (Volpe) -* Alexander Oberg (Volpe) -* Olivia Gillham (Volpe) -* Gary Baker (Volpe) -* Dr. Scott B. Smith (Volpe) -* Amy Vogel (Volpe) -* Amro El-Adle (Volpe) -* Kirby Ledvina (Volpe) -* Kevin Zhang (Volpe) -* Michelle Gilmore (Volpe) -* Mark Mockett (Volpe) - -## Project Sponsors: -The development of FTOT that contributed to this public version was funded by the U.S. Federal Aviation Administration (FAA) Office of Environment and Energy and the Department of Defense (DOD) Office of Naval Research through Interagency Agreements (IAA) FA4SCJ and FB48CS under the supervision of FAA’s Nathan Brown and by the U.S. Department of Energy (DOE) Office of Policy under IAA VXS3A2 under the supervision of Zachary Clement. Any opinions, findings, conclusions or recommendations expressed in this material are those of the authors and do not necessarily reflect the views of the FAA nor of DOE. - -## Acknowledgements: -The FTOT team thanks our Beta testers and collaborators for valuable input during the FTOT Public Release beta testing, including Dane Camenzind, Kristin Brandt, and Mike Wolcott (Washington State University), Mik Dale (Clemson University), Emily Newes and Ling Tao (National Renewable Energy Laboratory), Seckin Ozkul, Robert Hooker, and George Philippides (Univ. of South Florida), and Chris Ringo (Oregon State University). - -## License: -This project is licensed under the terms of the FTOT End User License Agreement. Please read it carefully. +# FTOT +Freight and Fuel Transportation Optimization Tool + +## Description: +FTOT is a flexible scenario-testing tool that optimizes the transportation of materials for future energy and freight scenarios. FTOT models and tracks commodity-specific information and can take into account conversion of raw materials to products (e.g., crude oil to jet fuel and diesel) and the fulfillment of downstream demand. FTOT was developed at the US Dept. of Transportation's Volpe National Transportation Systems Center. + +## Installation: +See [FTOT Installation wiki](https://github.com/VolpeUSDOT/FTOT-Public/wiki/FTOT-Installation-Guide) for the latest detailed instructions. If you need to run a legacy copy of FTOT compatible with ArcGIS rather than ArcGIS Pro, consult these [separate legacy instructions](https://github.com/VolpeUSDOT/FTOT-Public/wiki/FTOT-Installation-Guide-(ArcGIS-Legacy)) +* FTOT is a Python based tool. +* Clone or download the repository. +* Install the required dependencies (including ESRI ArcGIS Pro). +* Download the [documentation and scenario dataset](https://github.com/VolpeUSDOT/FTOT-Public/wiki/Documentation-and-Scenario-Datasets). + +## Usage: +* Usage is explained in the Quick Start documentation here: [Documentation and Scenario Dataset Wiki](https://github.com/VolpeUSDOT/FTOT-Public/wiki/Documentation-and-Scenario-Datasets) + +## Contributing: +* Add bugs and feature requests to the Issues tab for the Volpe Development Team to triage. + +## Credits: +* Dr. Kristin Lewis (Volpe) +* Matthew Pearlson (Volpe) +* Alexander Oberg (Volpe) +* Olivia Gillham (Volpe) +* Gary Baker (Volpe) +* Dr. Scott B. Smith (Volpe) +* Amy Vogel (Volpe) +* Amro El-Adle (Volpe) +* Kirby Ledvina (Volpe) +* Kevin Zhang (Volpe) +* Michelle Gilmore (Volpe) +* Mark Mockett (Volpe) + +## Project Sponsors: +The development of FTOT that contributed to this public version was funded by the U.S. Federal Aviation Administration (FAA) Office of Environment and Energy and the Department of Defense (DOD) Office of Naval Research through Interagency Agreements (IAA) FA4SCJ and FB48CS under the supervision of FAA’s Nathan Brown and by the U.S. Department of Energy (DOE) Office of Policy under IAA VXS3A2 under the supervision of Zachary Clement. Any opinions, findings, conclusions or recommendations expressed in this material are those of the authors and do not necessarily reflect the views of the FAA nor of DOE. + +## Acknowledgements: +The FTOT team thanks our Beta testers and collaborators for valuable input during the FTOT Public Release beta testing, including Dane Camenzind, Kristin Brandt, and Mike Wolcott (Washington State University), Mik Dale (Clemson University), Emily Newes and Ling Tao (National Renewable Energy Laboratory), Seckin Ozkul, Robert Hooker, and George Philippides (Univ. of South Florida), and Chris Ringo (Oregon State University). + +## License: +This project is licensed under the terms of the FTOT End User License Agreement. Please read it carefully. diff --git a/changelog.md b/changelog.md index 19ef470..c6efbe2 100644 --- a/changelog.md +++ b/changelog.md @@ -1,177 +1,182 @@ -# FTOT Change Log - -## v2021_3 -The 2021.3 release adds functionality enhancements related to the scenario XML file and artificial links, along with more detailed documentation and user exercises for importing facility location data, capacity scenarios, and artificial links. The following changes have been made: -- A major update to the scenario XML schema: (a) includes optional elements related to disruption, schedules, commodity mode, and network density reduction by default, (b) allows users to customize units for more elements, (c) removes unused elements, and (d) adds elements for rail and water short haul penalties (previously hardcoded and not customizable). This is a breaking change—previous versions of the XML schema will no longer work with FTOT version 2021.3. -- Additional reporting of artificial links for each facility and mode has been added. A new artificial_links.csv file stored in the debug folder of the scenario reports the length in miles of each artificial link created, or reports “NA” if no artificial link was able to connect the facility to the modal network. Stranded facilities (e.g., those not connected to the multimodal network at all) are not reported in this file. -- New Quick Start examples have been added to walk the user through FTOT scenarios involving capacity availability and artificial link distances. -- The default truck CO2 emission factors and fuel efficiency have been updated based on data from EPA’s MOVES3 model. -- The Tableau dashboard has been updated. The “Supply Chain Summary” page now displays a more complete picture of the scenario’s input data with the capability to scale and filter facilities by commodity. Facility tooltips also list the user-specified commodities and quantities for a facility. In addition, minor enhancements to the dashboard include (a) corrections to other legend scaling and tooltip displays, (b) relabeling of Scenario Cost to Dollar cost on the “By Commodity & Mode” page to match FTOT reporting, and (c) a reordering of the FTOT steps presented on the “Runtimes” page. -- Fixed a bug in which the value for max_processor_input in the processors csv file was assigned the default scenario units instead of the processor input file’s units. -- Updated the bat tool: (a) added FTOT’s pre-set file locations to reduce the user inputs requested and (b) updated the v5 .bat file naming convention to v6. - - -## v2021_2 -The 2021.2 release adds multiple functionality enhancements related to disruption, vehicle types, and routing. The following changes have been made: -- New functionality allowing the user to fully disrupt FTOT network segments (make segments unavailable) by leveraging a configurable disruption csv. -- An FTOT supporting tool automating the process of generating a disruption csv based on gridded exposure data (e.g., flooding data, HAZUS scenario outputs) has also been added. -- Ability to create custom vehicle types and assign them to individual commodities to enhance reporting. This feature introduces a new vehicle_types.csv file within FTOT and updates the functionality of the optional commodity_mode.csv input file. -- The network density reduction (NDR) functionality for identifying shortest paths (introduced in the 2020.4 release) has been refined to consider both phase of matter and commodity. With this additional development, NDR provides the optimal solution for all cases for which the functionality is allowed. The network presolve step for NDR remains disabled by default and is not compatible with candidate generation, the inclusion of modal capacity, and maximum allowable transport distance scenarios. -- Additional reporting of routes (shortest paths) included in the optimal solution has been added for scenarios where NDR has been enabled. A new optimal_routes.csv file stored in the debug folder of the scenario reports the starting and ending facility of each route, along with its routing cost and length in miles. -- The default background map for the Tableau dashboard has been changed from Dark to Streets to make it easier for the user to identify the local geography and path of the optimal solution. - -## v2021_1 - -The 2021.1 release finalizes the transition to Python 3, migrating from a dependency on ArcGIS 10.x to a dependency on ArcGIS Pro. The following changes have been made: -- Simplified and streamlined setup/installation instructions. -- Creation of a dedicated FTOT Python 3 environment within the FTOT installation directory. -- Due to the migration from ArcGIS to ArcGIS Pro, there is an updated backend process to outputting FTOT scenario maps. The ftot_maps.mxd mapping template provided with FTOT has been replaced with an ftot_maps.aprx project template compatible with ArcGIS Pro. - -## v2020_4_1 -The 2020.4.1 service pack release adds the ability to turn the network presolve step included in the -2020.4 release on or off through an optional network density reduction (NDR) function. The functionality -is controlled by the NDR_On parameter in the scenario configuration file; the default is for the step -to be disabled. Including the parameter in the scenario configuration file and setting it to True enables -the network presolve step described below in v2020_4. An exercise has been added to Quick Start 6 to -demonstrate the functionality and use cases of the NDR function. Additionally, the service pack release -includes improvements to the shortest path algorithm used by the network presolve step. Finally, small -fixes to handle unit conversion were made. - -## v2020_4 -The 2020.4 release includes a network presolve step that identifies links used in the shortest path -between sources and targets (e.g. RMP‐>Dest, RMP‐>Proc, Proc‐>Dest). It is enabled by default for -all scenarios except candidate generation and capacity. The presolve reduces the complexity of the -optimization and enhances performance. Mutli‐commodity input and Processor facilities were -enhanced to constrain the maximum capacity of the facility. Additionally, the release also includes -changes throughout for forward compatibility with Python 3.x in the future. Note however, that -2020.4 still requires Python 2.7 because of the ArcPy module dependency. Finally, several -miscellaneous fixes to improve the quality of the code base. - -GENERAL -- Removed remaining references to xtot_objects module. -- Added Python3 forward compatibility and code cleanup -- Added compatibility for ArcGIS basic license for certain operations in ftot.py, -ftot_postprocess.py, and ftot_routing.py -- The network is presolved for each appropriate source and target (e.g. Origin‐Destination pair) -using the shortest path algorithm in the NetworkX module. The links used in the shortest path -are stored in the shortest_path table in the main.db and used to speed up edge creation in the -O1 step and solving the optimization problem in the O2 step. - -PULP MODULE -- The optimization problem only includes edges created for network links identified in the -shortest path network presolve, except for scenarios that include candidate generation or -capacity. -- Multi‐commodity inputs for processors are properly constrained using a max_processor_input -field in the proc.csv input files. - -TABLEAU DASHBOARDS -- Commas are removed from the scenario name to prevent errors when parsing the Tableau -report CSV file. - -MAPS -- No changes to report in 2020.4 - -## v2020_3 - -The 2020.3 release includes a new setup script for installing FTOT and additional software dependencies, performance enhancements to reduce the memory footprint of scenarios at run time, and several miscellaneous fixes to improve the quality of the code base. - -GENERAL -- Added ArcGIS 10.8 and 10.8.1 to the list of compatible versions in ftot.py -- Created setup script for automating the installation of FTOT 2020.3 and Python dependencies -- Included the GDAL .whl files for installation in a new Dependencies folder -- Deprecated the use of Pickle in the post-optimization step -- Reduced the memory consumption in the G step by ignoring the geometry attributes of the temporary shape files. -- Moved the shape file export from the end of the C step to the beginning of the G step. -- Deletes the temporary networkx shape file directory from at the end of the G step. -- Renamed the post_optimization_64_bit() method - -PULP MODULE -- Added schedule functionality at the facility level, and Quick Start 2 Exercise 4 for as example for how to use the new schedule feature. - -TABLEAU DASHBOARDS -- Added units to tooltips throughout -- Changed color scheme for commodity and scenario name symbols -- Changed the alias of _MAP SYBOLOGY CALC to “Symbol By” -- Fixed bug where the story would open up empty because the scenario name filters were not selected. -- Fixed bug where the summary results by scenario name did not inherit the scenario name color scheme. -- Updated dashboard sizes to support non-1080p resolutions (e.g. 4k) -- Added absolute, percent difference, and difference views for the Summary graphics. -- Facilities map now includes the optimal and non-optimal facility filters. - -MAPS -- No changes to report in 2020.3 - - -## v2020_2 -The 2020.2 release includes a refactoring of the optimization module for candidate processor scenarios, enhancements to the Tableau dashboard functionality, and several general fixes for stability and user experience improvements. - -PULP MODULE REFACTORING -- Code is simplified to reduce redundancy between OC step and O1/O2 steps. - - Removed the following methods from ftot_pulp_candidate_generation.py: - - generate_all_vertices_candidate_gen - - generate_first_edges_from_source_facilities - - generate_connector_and_storage_edges - - add_storage_routes - - generate_first_edges_from_source_facilities - - set_edges_volume_capacity - - create_constraint_max_route_capacity -- Removed calls to serialize the optimization problem with pickle before the call to prob.solve(). -- Removed extra logger.debug messages from O1 and O2 steps. -- Removed redundant calls in pre_setup_pulp() to generate_first_edges_from_source_facilities and generate_all_edges_from_source_facilities in ftot_pulp.py - -TABLEAU DASHBOARDS -- Created Tableau Story to step through scenario dashboards. -- Added legend options to color routes by scenario name or commodity in comparison dashboards. -- Added fuel burn to scenario results. -- Added high-level scenario comparison summary. - -MAPS -- No changes to report in 2020.2 - -GENERAL -- Removed check for ArcGIS Network_analyst extension in ftot.py -- Updated installation instructions to specify version number for dependencies: pulp v.1.6.10, and imageio v.2.6 -- Fixed input validation bug for candidate facilities that crashed at runtime. -- Fixed Divide By Zero bug in Minimum Bounding Geometry step when road network is empty. -- Added FTOT Tool to batch update configuration parameters recursively in scenario.xml files -## v2020_1 -TABLEAU DASHBOARDS -- New Scenario Compare Workbook for comparing multiple FTOT scenarios in one Tableau workbook. -- Scenario Compare utility updated in FTOT Tools.py - now concatenates multiple scenarios and creates a new packaged Tableau workbook. -- Cost and volume numbers buffered to prevent overlap with mode graphics -- Facilities map tool tips no longer display metadata. -- Changes the config output from the S step (first one) to O2 step. -- Pipeline symbology fixes -- Runtimes now reported in the tableau report - -MAPS -- Basemap capability added -- Custom user created map capability added -- Extent functionality for maps changed -- Maps that show aggregate flows have been deprecated and removed -- Some map names have changed. In particular, the basemap name has been added to each of the maps. The numbering of a few maps was also changed and two had their titles shortened (e.g. raw_material_producer became rmp) for consistency with other maps. - -GENERAL -- Documentation updated to reflect the rail and water short-haul penalty. -- Documentation updated and Quick Start Scenarios created to demonstrate pipeline movements with the commodity_mode.csv -- Fixed hundredweight units typo in the python module used to track and convert units, Pint. -- General FTOT input validation, especially against whitespace which could cause optimization to fail. - -## v2019_4 -- arcgis 10.7.1 compatibility fixed. -- reports fail with logged exceptions fixed. -- updated minimum bounding geometry algorithm in the C step. -- total scenario cost is reported twice fixed. -- change log file added to repository. - -## v2019_3 -- processor facilities can now handle two input commodities. -- commodity-mode specific restrictions are enabled. -- tableau dashboard: display all facility types at once by default. -- emission factors are updated to reflect 2019 values. -- getter and setter definitions in the ftot_scenario.py module. -- commodity names are now case insensitive for input CSV files. -- optimizer passes back nearly zero values and causes bad maps. -- D Step throws an exception when no flow optimal solution. -- F step throws an exception if there is extra blank space in an input CSV file. -- tableau dashboard: units label for material moved by commodity and mode. +# FTOT Change Log + +## v2021_4 +The 2021.4 release provides updates related to emissions reporting, transport costs, as well as enhancements to output files and runtime. The following changes have been made: +- Expanded emissions report: Users can now generate a separate emissions report with total emissions by commodity and mode for seven non-CO2 pollutants (CO, CO2e, CH4, N2O, PM10, PM2.5, and VOC). For this feature, FTOT includes a set of default non-CO2 emission factors for road which users can update. The CO2 emissions factor is still reported in the main FTOT outputs and the emissions factor can be adjusted in the XML. +- User-adjustable density conversion factor: An optional density conversion factor has been added to the scenario XML for calculating emissions for liquid commodities on rail, water, and pipeline modes. +- Additions to the scenario XML schema: (a) includes optional elements related to emissions reporting and a density conversion factor, (b) updates base transport costs for road, rail, and water modes based on most recent data available from BTS. +- Reporting outputs: The reports folder generated by FTOT has been streamlined. The primary FTOT outputs (excluding maps) have been consolidated in a timestamped reports folder for each scenario run. This includes the Tableau dashboard, the CSV file report, the main text report, and any supplementary text reports (e.g., artificial links summary, optimal routes, expanded emissions reporting). +- Runtime improvements: Runtime of the post-processing step in FTOT has been significantly reduced for scenarios with a large number of raw material producer facilities. + +## v2021_3 +The 2021.3 release adds functionality enhancements related to the scenario XML file and artificial links, along with more detailed documentation and user exercises for importing facility location data, capacity scenarios, and artificial links. The following changes have been made: +- A major update to the scenario XML schema: (a) includes optional elements related to disruption, schedules, commodity mode, and network density reduction by default, (b) allows users to customize units for more elements, (c) removes unused elements, and (d) adds elements for rail and water short haul penalties (previously hardcoded and not customizable). This is a breaking change—previous versions of the XML schema will no longer work with FTOT version 2021.3. +- Additional reporting of artificial links for each facility and mode has been added. A new artificial_links.csv file stored in the debug folder of the scenario reports the length in miles of each artificial link created, or reports “NA” if no artificial link was able to connect the facility to the modal network. Stranded facilities (e.g., those not connected to the multimodal network at all) are not reported in this file. +- New Quick Start examples have been added to walk the user through FTOT scenarios involving capacity availability and artificial link distances. +- The default truck CO2 emission factors and fuel efficiency have been updated based on data from EPA’s MOVES3 model. +- The Tableau dashboard has been updated. The “Supply Chain Summary” page now displays a more complete picture of the scenario’s input data with the capability to scale and filter facilities by commodity. Facility tooltips also list the user-specified commodities and quantities for a facility. In addition, minor enhancements to the dashboard include (a) corrections to other legend scaling and tooltip displays, (b) relabeling of Scenario Cost to Dollar cost on the “By Commodity & Mode” page to match FTOT reporting, and (c) a reordering of the FTOT steps presented on the “Runtimes” page. +- Fixed a bug in which the value for max_processor_input in the processors csv file was assigned the default scenario units instead of the processor input file’s units. +- Updated the bat tool: (a) added FTOT’s pre-set file locations to reduce the user inputs requested and (b) updated the v5 .bat file naming convention to v6. + +## v2021_2 +The 2021.2 release adds multiple functionality enhancements related to disruption, vehicle types, and routing. The following changes have been made: +- New functionality allowing the user to fully disrupt FTOT network segments (make segments unavailable) by leveraging a configurable disruption csv. +- An FTOT supporting tool automating the process of generating a disruption csv based on gridded exposure data (e.g., flooding data, HAZUS scenario outputs) has also been added. +- Ability to create custom vehicle types and assign them to individual commodities to enhance reporting. This feature introduces a new vehicle_types.csv file within FTOT and updates the functionality of the optional commodity_mode.csv input file. +- The network density reduction (NDR) functionality for identifying shortest paths (introduced in the 2020.4 release) has been refined to consider both phase of matter and commodity. With this additional development, NDR provides the optimal solution for all cases for which the functionality is allowed. The network presolve step for NDR remains disabled by default and is not compatible with candidate generation, the inclusion of modal capacity, and maximum allowable transport distance scenarios. +- Additional reporting of routes (shortest paths) included in the optimal solution has been added for scenarios where NDR has been enabled. A new optimal_routes.csv file stored in the debug folder of the scenario reports the starting and ending facility of each route, along with its routing cost and length in miles. +- The default background map for the Tableau dashboard has been changed from Dark to Streets to make it easier for the user to identify the local geography and path of the optimal solution. + +## v2021_1 +The 2021.1 release finalizes the transition to Python 3, migrating from a dependency on ArcGIS 10.x to a dependency on ArcGIS Pro. The following changes have been made: +- Simplified and streamlined setup/installation instructions. +- Creation of a dedicated FTOT Python 3 environment within the FTOT installation directory. +- Due to the migration from ArcGIS to ArcGIS Pro, there is an updated backend process to outputting FTOT scenario maps. The ftot_maps.mxd mapping template provided with FTOT has been replaced with an ftot_maps.aprx project template compatible with ArcGIS Pro. + +## v2020_4_1 +The 2020.4.1 service pack release adds the ability to turn the network presolve step included in the +2020.4 release on or off through an optional network density reduction (NDR) function. The functionality +is controlled by the NDR_On parameter in the scenario configuration file; the default is for the step +to be disabled. Including the parameter in the scenario configuration file and setting it to True enables +the network presolve step described below in v2020_4. An exercise has been added to Quick Start 6 to +demonstrate the functionality and use cases of the NDR function. Additionally, the service pack release +includes improvements to the shortest path algorithm used by the network presolve step. Finally, small +fixes to handle unit conversion were made. + +## v2020_4 +The 2020.4 release includes a network presolve step that identifies links used in the shortest path +between sources and targets (e.g. RMP‐>Dest, RMP‐>Proc, Proc‐>Dest). It is enabled by default for +all scenarios except candidate generation and capacity. The presolve reduces the complexity of the +optimization and enhances performance. Mutli‐commodity input and Processor facilities were +enhanced to constrain the maximum capacity of the facility. Additionally, the release also includes +changes throughout for forward compatibility with Python 3.x in the future. Note however, that +2020.4 still requires Python 2.7 because of the ArcPy module dependency. Finally, several +miscellaneous fixes to improve the quality of the code base. + +GENERAL +- Removed remaining references to xtot_objects module. +- Added Python3 forward compatibility and code cleanup +- Added compatibility for ArcGIS basic license for certain operations in ftot.py, +ftot_postprocess.py, and ftot_routing.py +- The network is presolved for each appropriate source and target (e.g. Origin‐Destination pair) +using the shortest path algorithm in the NetworkX module. The links used in the shortest path +are stored in the shortest_path table in the main.db and used to speed up edge creation in the +O1 step and solving the optimization problem in the O2 step. + +PULP MODULE +- The optimization problem only includes edges created for network links identified in the +shortest path network presolve, except for scenarios that include candidate generation or +capacity. +- Multi‐commodity inputs for processors are properly constrained using a max_processor_input +field in the proc.csv input files. + +TABLEAU DASHBOARDS +- Commas are removed from the scenario name to prevent errors when parsing the Tableau +report CSV file. + +MAPS +- No changes to report in 2020.4 + +## v2020_3 +The 2020.3 release includes a new setup script for installing FTOT and additional software dependencies, performance enhancements to reduce the memory footprint of scenarios at run time, and several miscellaneous fixes to improve the quality of the code base. + +GENERAL +- Added ArcGIS 10.8 and 10.8.1 to the list of compatible versions in ftot.py +- Created setup script for automating the installation of FTOT 2020.3 and Python dependencies +- Included the GDAL .whl files for installation in a new Dependencies folder +- Deprecated the use of Pickle in the post-optimization step +- Reduced the memory consumption in the G step by ignoring the geometry attributes of the temporary shape files. +- Moved the shape file export from the end of the C step to the beginning of the G step. +- Deletes the temporary networkx shape file directory from at the end of the G step. +- Renamed the post_optimization_64_bit() method + +PULP MODULE +- Added schedule functionality at the facility level, and Quick Start 2 Exercise 4 for as example for how to use the new schedule feature. + +TABLEAU DASHBOARDS +- Added units to tooltips throughout +- Changed color scheme for commodity and scenario name symbols +- Changed the alias of _MAP SYBOLOGY CALC to “Symbol By” +- Fixed bug where the story would open up empty because the scenario name filters were not selected. +- Fixed bug where the summary results by scenario name did not inherit the scenario name color scheme. +- Updated dashboard sizes to support non-1080p resolutions (e.g. 4k) +- Added absolute, percent difference, and difference views for the Summary graphics. +- Facilities map now includes the optimal and non-optimal facility filters. + +MAPS +- No changes to report in 2020.3 + +## v2020_2 +The 2020.2 release includes a refactoring of the optimization module for candidate processor scenarios, enhancements to the Tableau dashboard functionality, and several general fixes for stability and user experience improvements. + +PULP MODULE REFACTORING +- Code is simplified to reduce redundancy between OC step and O1/O2 steps. + - Removed the following methods from ftot_pulp_candidate_generation.py: + - generate_all_vertices_candidate_gen + - generate_first_edges_from_source_facilities + - generate_connector_and_storage_edges + - add_storage_routes + - generate_first_edges_from_source_facilities + - set_edges_volume_capacity + - create_constraint_max_route_capacity +- Removed calls to serialize the optimization problem with pickle before the call to prob.solve(). +- Removed extra logger.debug messages from O1 and O2 steps. +- Removed redundant calls in pre_setup_pulp() to generate_first_edges_from_source_facilities and generate_all_edges_from_source_facilities in ftot_pulp.py + +TABLEAU DASHBOARDS +- Created Tableau Story to step through scenario dashboards. +- Added legend options to color routes by scenario name or commodity in comparison dashboards. +- Added fuel burn to scenario results. +- Added high-level scenario comparison summary. + +MAPS +- No changes to report in 2020.2 + +GENERAL +- Removed check for ArcGIS Network_analyst extension in ftot.py +- Updated installation instructions to specify version number for dependencies: pulp v.1.6.10, and imageio v.2.6 +- Fixed input validation bug for candidate facilities that crashed at runtime. +- Fixed Divide By Zero bug in Minimum Bounding Geometry step when road network is empty. +- Added FTOT Tool to batch update configuration parameters recursively in scenario.xml files + +## v2020_1 +TABLEAU DASHBOARDS +- New Scenario Compare Workbook for comparing multiple FTOT scenarios in one Tableau workbook. +- Scenario Compare utility updated in FTOT Tools.py - now concatenates multiple scenarios and creates a new packaged Tableau workbook. +- Cost and volume numbers buffered to prevent overlap with mode graphics +- Facilities map tool tips no longer display metadata. +- Changes the config output from the S step (first one) to O2 step. +- Pipeline symbology fixes +- Runtimes now reported in the tableau report + +MAPS +- Basemap capability added +- Custom user created map capability added +- Extent functionality for maps changed +- Maps that show aggregate flows have been deprecated and removed +- Some map names have changed. In particular, the basemap name has been added to each of the maps. The numbering of a few maps was also changed and two had their titles shortened (e.g. raw_material_producer became rmp) for consistency with other maps. + +GENERAL +- Documentation updated to reflect the rail and water short-haul penalty. +- Documentation updated and Quick Start Scenarios created to demonstrate pipeline movements with the commodity_mode.csv +- Fixed hundredweight units typo in the python module used to track and convert units, Pint. +- General FTOT input validation, especially against whitespace which could cause optimization to fail. + +## v2019_4 +- arcgis 10.7.1 compatibility fixed. +- reports fail with logged exceptions fixed. +- updated minimum bounding geometry algorithm in the C step. +- total scenario cost is reported twice fixed. +- change log file added to repository. + +## v2019_3 +- processor facilities can now handle two input commodities. +- commodity-mode specific restrictions are enabled. +- tableau dashboard: display all facility types at once by default. +- emission factors are updated to reflect 2019 values. +- getter and setter definitions in the ftot_scenario.py module. +- commodity names are now case insensitive for input CSV files. +- optimizer passes back nearly zero values and causes bad maps. +- D Step throws an exception when no flow optimal solution. +- F step throws an exception if there is extra blank space in an input CSV file. +- tableau dashboard: units label for material moved by commodity and mode. diff --git a/program/ftot.py b/program/ftot.py index fbff80c..4e999a4 100644 --- a/program/ftot.py +++ b/program/ftot.py @@ -1,314 +1,315 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot.py -# -# Purpose: This module initiates an FTOT run, parses the scenario xml into a scenario object -# and calls the appropriate tasks based on user supplied arguments. -# -# --------------------------------------------------------------------------------------------------- - -import sys -import os -import argparse -import ftot_supporting -import traceback -import datetime - -import pint -from pint import UnitRegistry - -ureg = UnitRegistry() -Q_ = ureg.Quantity -ureg.define('usd = [currency]') # add the US Dollar, "USD" to the unit registry -# solves issue in pint 0.9 -if pint.__version__ == 0.9: - ureg.define('short_hundredweight = short_hunderdweight') - ureg.define('long_hundredweight = long_hunderdweight') - ureg.define('us_ton = US_ton') - - -FTOT_VERSION = "2021.3" -SCHEMA_VERSION = "6.0.0" -VERSION_DATE = "10/5/2021" - -# =================================================================================================== - -if __name__ == '__main__': - - start_time = datetime.datetime.now() - - # PARSE ARGS - # ---------------------------------------------------------------------------------------------- - - program_description = 'Freight/Fuels Transportation Optimization Tool (FTOT). Version Number: ' \ - + FTOT_VERSION + ", (" + VERSION_DATE + ")" - - help_text = """ - The command-line input expected for this script is as follows: - - TheFilePathOfThisScript FullPathToXmlScenarioConfigFile TaskToRun - - Valid values of Task to run include: - - # pre-optimization options - # ----------------------- - s = setup; deletes existing database and geodatabase, creates new database, and copies base network to - scenario geodatabase - f = facilities; process GIS feature classes and commodity input CSV files - c = connectivity; connects facilities to the network using artificial links and exports geodatabase FCs as - shapefiles - g = graph; reads in shapefiles using networkX and prepares, cleans, and stores the network digraph in the - database - - # optimization options - # -------------------- - o1 = optimization setup; structures tables necessary for optimization run - - o2 = optimization calculation; Calculates the optimal flow and unmet demand for each OD pair with a route - - o2b = optional step to solve and save pulp problem from pickled, constrained, problem - - oc1-3 = optimization candidate generation; optional step to create candidate processors in the GIS and DB - based off the FTOT v5 candidate generation algorithm optimization results - - oc2b = optional step to solve and save pulp problem from pickled, constrained, problem. Run oc3 after. - - os = optimization sourcing; optional step to calculate source facilities for optimal flows, - uses existing solution from o and must be run after o - - # post-optimization options - # ------------------------- - - f2 = facilities; same as f, used for candidate generation and labeled differently for reporting purposes - c2 = connectivity; same as c, used for candidate generation and labeled differently for reporting purposes - g2 = graph; same as g, used for candidate generation and labeled differently for reporting purposes - - # reporting and mapping - # --------------------- - p = post-processing of optimal solution and reporting preparation - d = create data reports - m = create map documents with simple basemap - mb = optionally create map documents with a light gray basemap - mc = optionally create map documents with a topographic basemap - md = optionally create map documents with a streets basemap - m2 = time and commodity mapping with simple basemap - m2b = optionally create time and commodity mapping with a light gray basemap - m2c = optionally create time and commodity mapping with a topographic basemap - m2d = optionally create time and commodity mapping with a streets basemap - - # utilities, tools, and advanced options - # --------------------------------------- - test = a test method that can be used for debugging purposes - --skip_arcpy_check = a command line option to skip the arcpy dependency check - """ - - parser = argparse.ArgumentParser(description=program_description, usage=help_text) - - parser.add_argument("config_file", help="The full path to the XML Scenario", type=str) - - parser.add_argument("task", choices=("s", "f", "f2", "c", "c2", "g", "g2", - "o", "oc", - "o1", "o2", "o2b", "oc1", "oc2", "oc2b", "oc3", "os", "p", - "d", "m", "mb", "mc", "md", "m2", "m2b", "m2c", "m2d" - "test" - ), type=str) - parser.add_argument('-skip_arcpy_check', action='store_true', - default=False, - dest='skip_arcpy_check', - help='Use argument to skip the arcpy dependency check') - - args = parser.parse_args() - - if len(sys.argv) >= 3: - args = parser.parse_args() - else: - parser.print_help() - sys.exit() - - if os.path.exists(args.config_file): - ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) - else: - print ("{} doesn't exist".format(args.config_file)) - sys.exit() - - # set up logging and report run start time - # ---------------------------------------------------------------------------------------------- - xml_file_location = os.path.abspath(args.config_file) - from ftot_supporting import create_loggers - - logger = create_loggers(os.path.dirname(xml_file_location), args.task) - - logger.info("=================================================================================") - logger.info("============= FTOT RUN STARTING. Run Option = {:2} ===============================".format(str(args.task).upper())) - logger.info("=================================================================================") - - # load the ftot scenario - # ---------------------------------------------------------------------------------------------- - - from ftot_scenario import * - - xml_schema_file_location = os.path.join(ftot_program_directory, "lib", "Master_FTOT_Schema.xsd") - - if not os.path.exists(xml_schema_file_location): - logger.error("can't find xml schema at {}".format(xml_schema_file_location)) - sys.exit() - - logger.debug("start: load_scenario_config_file") - the_scenario = load_scenario_config_file(xml_file_location, xml_schema_file_location, logger) - - # write scenario configuration to the log for incorporation in the report - # ----------------------------------------------------------------------- - dump_scenario_info_to_report(the_scenario, logger) - - if args.task in ['s', 'sc']: - create_scenario_config_db(the_scenario, logger) - else: - check_scenario_config_db(the_scenario, logger) - - # check that arcpy is available if the -skip_arcpy_check flag is not set - # ---------------------------------------------------------------------- - if not args.skip_arcpy_check: - try: - import arcpy - arcgis_pro_version = arcpy.GetInstallInfo()['Version'] - if float(arcgis_pro_version[0:3]) < 2.6: - logger.error("Version {} of ArcGIS Pro is not supported. Exiting.".format(arcgis_pro_version)) - sys.exit() - - except RuntimeError: - logger.error("ArcGIS Pro 2.6 or later is required to run this script. If you do have ArcGIS Pro installed, " - "confirm that it is properly licensed and/or that the license manager is accessible. Exiting.") - sys.exit() - - # check that pulp is available - # ---------------------------------------------------------------------------------------------- - - try: - from pulp import * - except ImportError: - logger.error("This script requires the PuLP LP modeler to optimize the routing. " - "Download the modeler here: http://code.google.com/p/pulp-or/downloads/list.") - sys.exit() - - # run the task - # ---------------------------------------------------------------------------------------------- - - try: - - # setup the scenario - if args.task in ['s', 'sc']: - from ftot_setup import setup - setup(the_scenario, logger) - - # facilities - elif args.task in ['f', 'f2']: - from ftot_facilities import facilities - facilities(the_scenario, logger) - - # connectivity; hook facilities into the network - elif args.task in ['c', 'c2']: - from ftot_routing import connectivity - connectivity(the_scenario, logger) - - # graph, convert GIS network and facility shapefiles into to a networkx graph - elif args.task in ['g', 'g2']: - from ftot_networkx import graph - graph(the_scenario, logger) - - # optimization - elif args.task in ['o']: - from ftot_pulp import o1, o2 - o1(the_scenario, logger) - o2(the_scenario, logger) - - # candidate optimization - elif args.task in ['oc']: - from ftot_pulp_candidate_generation import oc1, oc2, oc3 - oc1(the_scenario, logger) - oc2(the_scenario, logger) - oc3(the_scenario, logger) - - # optimization setup - elif args.task in ['o1']: - from ftot_pulp import o1 - o1(the_scenario, logger) - - # optimization solve - elif args.task in ['o2']: - from ftot_pulp import o2 - o2(the_scenario, logger) - - # optional step to solve and save pulp problem from pickle - elif args.task == 'o2b': - from ftot_pulp import o2b - o2b(the_scenario, logger) - - # optimization option - processor candidates generation - elif args.task == 'oc1': - from ftot_pulp_candidate_generation import oc1 - oc1(the_scenario, logger) - - # optimization option - processor candidates generation - elif args.task == 'oc2': - from ftot_pulp_candidate_generation import oc2 - oc2(the_scenario, logger) - - elif args.task == 'oc3': - from ftot_pulp_candidate_generation import oc3 - oc3(the_scenario, logger) - - # optional step to solve and save pulp problem from pickle - elif args.task == 'oc2b': - from ftot_pulp import oc2b - oc2b(the_scenario, logger) - - # post-optimization tracking for source based on existing optimal solution - elif args.task in ['os', 'bscoe']: - from ftot_pulp_sourcing import o_sourcing - o_sourcing(the_scenario, logger) - - # post-process the optimal solution - elif args.task in ["p", "p2"]: - from ftot_postprocess import route_post_optimization_db - route_post_optimization_db(the_scenario, logger) - - # data report - elif args.task == "d": - from ftot_report import generate_reports - generate_reports(the_scenario, logger) - - # currently m step has three basemap alternatives-- see key above - elif args.task in ["m", "mb", "mc", "md"]: - from ftot_maps import new_map_creation - new_map_creation(the_scenario, logger, args.task) - - # currently m2 step has three basemap alternatives-- see key above - elif args.task in ["m2", "m2b", "m2c", "m2d"]: - from ftot_maps import prepare_time_commodity_subsets_for_mapping - prepare_time_commodity_subsets_for_mapping(the_scenario, logger, args.task) - - elif args.task == "test": - logger.info("in the test case") - import pdb - pdb.set_trace() - - except: - - stack_trace = traceback.format_exc() - split_stack_trace = stack_trace.split('\n') - logger.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EXCEPTION RAISED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - for i in range(0, len(split_stack_trace)): - trace_line = split_stack_trace[i].rstrip() - if trace_line != "": # issue #182 - check if the line is blank. if it isn't, record it in the log. - logger.error(trace_line) - logger.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EXCEPTION RAISED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!") - - sys.exit(1) - - logger.info("======================== FTOT RUN FINISHED: {:2} ==================================".format( - str(args.task).upper())) - logger.info("======================== Total Runtime (HMS): \t{} \t ".format( - ftot_supporting.get_total_runtime_string(start_time))) - logger.info("=================================================================================") - logger.runtime( - "{} Step - Total Runtime (HMS): \t{}".format(args.task, ftot_supporting.get_total_runtime_string(start_time))) - logging.shutdown() +# --------------------------------------------------------------------------------------------------- +# Name: ftot.py +# +# Purpose: This module initiates an FTOT run, parses the scenario xml into a scenario object +# and calls the appropriate tasks based on user supplied arguments. +# +# --------------------------------------------------------------------------------------------------- + +import sys +import os +import argparse +import ftot_supporting +import traceback +import datetime + +import pint +from pint import UnitRegistry + +ureg = UnitRegistry() +Q_ = ureg.Quantity +ureg.define('usd = [currency]') # add the US Dollar, "USD" to the unit registry +# solves issue in pint 0.9 +if pint.__version__ == 0.9: + ureg.define('short_hundredweight = short_hunderdweight') + ureg.define('long_hundredweight = long_hunderdweight') + ureg.define('us_ton = US_ton') + + +FTOT_VERSION = "2021.4" +SCHEMA_VERSION = "6.0.1" +VERSION_DATE = "1/10/2022" + +# =================================================================================================== + +if __name__ == '__main__': + + start_time = datetime.datetime.now() + + # PARSE ARGS + # ---------------------------------------------------------------------------------------------- + + program_description = 'Freight/Fuels Transportation Optimization Tool (FTOT). Version Number: ' \ + + FTOT_VERSION + ", (" + VERSION_DATE + ")" + + help_text = """ + The command-line input expected for this script is as follows: + + TheFilePathOfThisScript FullPathToXmlScenarioConfigFile TaskToRun + + Valid values of Task to run include: + + # pre-optimization options + # ----------------------- + s = setup; deletes existing database and geodatabase, creates new database, and copies base network to + scenario geodatabase + f = facilities; process GIS feature classes and commodity input CSV files + c = connectivity; connects facilities to the network using artificial links and exports geodatabase FCs as + shapefiles + g = graph; reads in shapefiles using networkX and prepares, cleans, and stores the network digraph in the + database + + # optimization options + # -------------------- + o1 = optimization setup; structures tables necessary for optimization run + TODO pickle "prob" and move at least variable creation to o1, constraints if possible + + o2 = optimization calculation; Calculates the optimal flow and unmet demand for each OD pair with a route + + o2b = optional step to solve and save pulp problem from pickled, constrained, problem + + oc1-3 = optimization candidate generation; optional step to create candidate processors in the GIS and DB + based off the FTOT v5 candidate generation algorithm optimization results + + oc2b = optional step to solve and save pulp problem from pickled, constrained, problem. Run oc3 after. + + os = optimization sourcing; optional step to calculate source facilities for optimal flows, + uses existing solution from o and must be run after o + + # post-optimization options + # ------------------------- + + f2 = facilities; same as f, used for candidate generation and labeled differently for reporting purposes + c2 = connectivity; same as c, used for candidate generation and labeled differently for reporting purposes + g2 = graph; same as g, used for candidate generation and labeled differently for reporting purposes + + # reporting and mapping + # --------------------- + p = post-processing of optimal solution and reporting preparation + d = create data reports + m = create map documents with simple basemap + mb = optionally create map documents with a light gray basemap + mc = optionally create map documents with a topographic basemap + md = optionally create map documents with a streets basemap + m2 = time and commodity mapping with simple basemap + m2b = optionally create time and commodity mapping with a light gray basemap + m2c = optionally create time and commodity mapping with a topographic basemap + m2d = optionally create time and commodity mapping with a streets basemap + + # utilities, tools, and advanced options + # --------------------------------------- + test = a test method that can be used for debugging purposes + --skip_arcpy_check = a command line option to skip the arcpy dependency check + """ + + parser = argparse.ArgumentParser(description=program_description, usage=help_text) + + parser.add_argument("config_file", help="The full path to the XML Scenario", type=str) + + parser.add_argument("task", choices=("s", "f", "f2", "c", "c2", "g", "g2", + "o", "oc", + "o1", "o2", "o2b", "oc1", "oc2", "oc2b", "oc3", "os", "p", + "d", "m", "mb", "mc", "md", "m2", "m2b", "m2c", "m2d" + "test" + ), type=str) + parser.add_argument('-skip_arcpy_check', action='store_true', + default=False, + dest='skip_arcpy_check', + help='Use argument to skip the arcpy dependency check') + + args = parser.parse_args() + + if len(sys.argv) >= 3: + args = parser.parse_args() + else: + parser.print_help() + sys.exit() + + if os.path.exists(args.config_file): + ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) + else: + print ("{} doesn't exist".format(args.config_file)) + sys.exit() + + # set up logging and report run start time + # ---------------------------------------------------------------------------------------------- + xml_file_location = os.path.abspath(args.config_file) + from ftot_supporting import create_loggers + + logger = create_loggers(os.path.dirname(xml_file_location), args.task) + + logger.info("=================================================================================") + logger.info("============= FTOT RUN STARTING. Run Option = {:2} ===============================".format(str(args.task).upper())) + logger.info("=================================================================================") + + # load the ftot scenario + # ---------------------------------------------------------------------------------------------- + + from ftot_scenario import * + + xml_schema_file_location = os.path.join(ftot_program_directory, "lib", "Master_FTOT_Schema.xsd") + + if not os.path.exists(xml_schema_file_location): + logger.error("can't find xml schema at {}".format(xml_schema_file_location)) + sys.exit() + + logger.debug("start: load_scenario_config_file") + the_scenario = load_scenario_config_file(xml_file_location, xml_schema_file_location, logger) + + # write scenario configuration to the log for incorporation in the report + # ----------------------------------------------------------------------- + dump_scenario_info_to_report(the_scenario, logger) + + if args.task in ['s', 'sc']: + create_scenario_config_db(the_scenario, logger) + else: + check_scenario_config_db(the_scenario, logger) + + # check that arcpy is available if the -skip_arcpy_check flag is not set + # ---------------------------------------------------------------------- + if not args.skip_arcpy_check: + try: + import arcpy + arcgis_pro_version = arcpy.GetInstallInfo()['Version'] + if float(arcgis_pro_version[0:3]) < 2.6: + logger.error("Version {} of ArcGIS Pro is not supported. Exiting.".format(arcgis_pro_version)) + sys.exit() + + except RuntimeError: + logger.error("ArcGIS Pro 2.6 or later is required to run this script. If you do have ArcGIS Pro installed, " + "confirm that it is properly licensed and/or that the license manager is accessible. Exiting.") + sys.exit() + + # check that pulp is available + # ---------------------------------------------------------------------------------------------- + + try: + from pulp import * + except ImportError: + logger.error("This script requires the PuLP LP modeler to optimize the routing. " + "Download the modeler here: http://code.google.com/p/pulp-or/downloads/list.") + sys.exit() + + # run the task + # ---------------------------------------------------------------------------------------------- + + try: + + # setup the scenario + if args.task in ['s', 'sc']: + from ftot_setup import setup + setup(the_scenario, logger) + + # facilities + elif args.task in ['f', 'f2']: + from ftot_facilities import facilities + facilities(the_scenario, logger) + + # connectivity; hook facilities into the network + elif args.task in ['c', 'c2']: + from ftot_routing import connectivity + connectivity(the_scenario, logger) + + # graph, convert GIS network and facility shapefiles into to a networkx graph + elif args.task in ['g', 'g2']: + from ftot_networkx import graph + graph(the_scenario, logger) + + # optimization + elif args.task in ['o']: + from ftot_pulp import o1, o2 + o1(the_scenario, logger) + o2(the_scenario, logger) + + # candidate optimization + elif args.task in ['oc']: + from ftot_pulp_candidate_generation import oc1, oc2, oc3 + oc1(the_scenario, logger) + oc2(the_scenario, logger) + oc3(the_scenario, logger) + + # optimization setup + elif args.task in ['o1']: + from ftot_pulp import o1 + o1(the_scenario, logger) + + # optimization solve + elif args.task in ['o2']: + from ftot_pulp import o2 + o2(the_scenario, logger) + + # optional step to solve and save pulp problem from pickle + elif args.task == 'o2b': + from ftot_pulp import o2b + o2b(the_scenario, logger) + + # optimization option - processor candidates generation + elif args.task == 'oc1': + from ftot_pulp_candidate_generation import oc1 + oc1(the_scenario, logger) + + # optimization option - processor candidates generation + elif args.task == 'oc2': + from ftot_pulp_candidate_generation import oc2 + oc2(the_scenario, logger) + + elif args.task == 'oc3': + from ftot_pulp_candidate_generation import oc3 + oc3(the_scenario, logger) + + # optional step to solve and save pulp problem from pickle + elif args.task == 'oc2b': + from ftot_pulp import oc2b + oc2b(the_scenario, logger) + + # post-optimization tracking for source based on existing optimal solution + elif args.task in ['os', 'bscoe']: + from ftot_pulp_sourcing import o_sourcing + o_sourcing(the_scenario, logger) + + # post-process the optimal solution + elif args.task in ["p", "p2"]: + from ftot_postprocess import route_post_optimization_db + route_post_optimization_db(the_scenario, logger) + + # data report + elif args.task == "d": + from ftot_report import generate_reports + generate_reports(the_scenario, logger) + + # currently m step has three basemap alternatives-- see key above + elif args.task in ["m", "mb", "mc", "md"]: + from ftot_maps import new_map_creation + new_map_creation(the_scenario, logger, args.task) + + # currently m2 step has three basemap alternatives-- see key above + elif args.task in ["m2", "m2b", "m2c", "m2d"]: + from ftot_maps import prepare_time_commodity_subsets_for_mapping + prepare_time_commodity_subsets_for_mapping(the_scenario, logger, args.task) + + elif args.task == "test": + logger.info("in the test case") + import pdb + pdb.set_trace() + + except: + + stack_trace = traceback.format_exc() + split_stack_trace = stack_trace.split('\n') + logger.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EXCEPTION RAISED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + for i in range(0, len(split_stack_trace)): + trace_line = split_stack_trace[i].rstrip() + if trace_line != "": # issue #182 - check if the line is blank. if it isn't, record it in the log. + logger.error(trace_line) + logger.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EXCEPTION RAISED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!") + + sys.exit(1) + + logger.info("======================== FTOT RUN FINISHED: {:2} ==================================".format( + str(args.task).upper())) + logger.info("======================== Total Runtime (HMS): \t{} \t ".format( + ftot_supporting.get_total_runtime_string(start_time))) + logger.info("=================================================================================") + logger.runtime( + "{} Step - Total Runtime (HMS): \t{}".format(args.task, ftot_supporting.get_total_runtime_string(start_time))) + logging.shutdown() diff --git a/program/ftot_facilities.py b/program/ftot_facilities.py index fe84d2b..9723ac6 100644 --- a/program/ftot_facilities.py +++ b/program/ftot_facilities.py @@ -1,1208 +1,1208 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot_facilities -# -# Purpose: setup for raw material producers (RMP) and ultimate destinations. -# adds demand for destinations. adds supply for RMP. -# hooks facilities into the network using artificial links. -# special consideration for how pipeline links are added. -# -# --------------------------------------------------------------------------------------------------- - -import ftot_supporting -import ftot_supporting_gis -import arcpy -import datetime -import os -import sqlite3 -from ftot import ureg, Q_ -from six import iteritems -LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') - - -# =============================================================================== - - -def facilities(the_scenario, logger): - gis_clean_fc(the_scenario, logger) - gis_populate_fc(the_scenario, logger) - - db_cleanup_tables(the_scenario, logger) - db_populate_tables(the_scenario, logger) - db_report_commodity_potentials(the_scenario, logger) - - if the_scenario.processors_candidate_slate_data != 'None': - # make candidate_process_list and candidate_process_commodities tables - from ftot_processor import generate_candidate_processor_tables - generate_candidate_processor_tables(the_scenario, logger) - - -# =============================================================================== - - -def db_drop_table(the_scenario, table_name, logger): - with sqlite3.connect(the_scenario.main_db) as main_db_con: - logger.debug("drop the {} table".format(table_name)) - main_db_con.execute("drop table if exists {};".format(table_name)) - - -# ============================================================================== - - -def db_cleanup_tables(the_scenario, logger): - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - # DB CLEAN UP - # ------------ - logger.info("start: db_cleanup_tables") - - # a new run is a new scenario in the main.db - # so drop and create the following tables if they exists - # -------------------------------------------- - - # locations table - logger.debug("drop the locations table") - main_db_con.execute("drop table if exists locations;") - logger.debug("create the locations table") - main_db_con.executescript( - "create table locations(location_ID INTEGER PRIMARY KEY, shape_x real, shape_y real, ignore_location text);") - - # tmp_facility_locations table - # a temp table used to map the facility_name from the facility_commodities data csv - # to the location_id. its used to populate the facility_commodities table - # and deleted after it is populated. - logger.debug("drop the tmp_facility_locations table") - main_db_con.execute("drop table if exists tmp_facility_locations;") - logger.debug("create the tmp_facility_locations table") - main_db_con.executescript( - "create table tmp_facility_locations(location_ID INTEGER , facility_name text PRIMARY KEY);") - - # facilities table - logger.debug("drop the facilities table") - main_db_con.execute("drop table if exists facilities;") - logger.debug("create the facilities table") - main_db_con.executescript( - "create table facilities(facility_ID INTEGER PRIMARY KEY, location_id integer, facility_name text, facility_type_id integer, ignore_facility text, candidate binary, schedule_id integer, max_capacity float);") - - # facility_type_id table - logger.debug("drop the facility_type_id table") - main_db_con.execute("drop table if exists facility_type_id;") - main_db_con.executescript("create table facility_type_id(facility_type_id integer primary key, facility_type text);") - - # phase_of_matter_id table - logger.debug("drop the phase_of_matter_id table") - main_db_con.execute("drop table if exists phase_of_matter_id;") - logger.debug("create the phase_of_matter_id table") - main_db_con.executescript( - "create table phase_of_matter_id(phase_of_matter_id INTEGER PRIMARY KEY, phase_of_matter text);") - - # facility_commodities table - # tracks the relationship of facility and commodities in or out - logger.debug("drop the facility_commodities table") - main_db_con.execute("drop table if exists facility_commodities;") - logger.debug("create the facility_commodities table") - main_db_con.executescript( - "create table facility_commodities(facility_id integer, location_id integer, commodity_id interger, " - "quantity numeric, units text, io text, share_max_transport_distance text);") - - # commodities table - logger.debug("drop the commodities table") - main_db_con.execute("drop table if exists commodities;") - logger.debug("create the commodities table") - main_db_con.executescript( - """create table commodities(commodity_ID INTEGER PRIMARY KEY, commodity_name text, supertype text, subtype text, - units text, phase_of_matter text, max_transport_distance numeric, proportion_of_supertype numeric, - share_max_transport_distance text, CONSTRAINT unique_name UNIQUE(commodity_name) );""") - # proportion_of_supertype specifies how much demand is satisfied by this subtype relative to the "pure" - # fuel/commodity. this will depend on the process - - # schedule_names table - logger.debug("drop the schedule names table") - main_db_con.execute("drop table if exists schedule_names;") - logger.debug("create the schedule names table") - main_db_con.executescript( - """create table schedule_names(schedule_id INTEGER PRIMARY KEY, schedule_name text);""") - - # schedules table - logger.debug("drop the schedules table") - main_db_con.execute("drop table if exists schedules;") - logger.debug("create the schedules table") - main_db_con.executescript( - """create table schedules(schedule_id integer, day integer, availability numeric);""") - - # coprocessing reference table - logger.debug("drop the coprocessing table") - main_db_con.execute("drop table if exists coprocessing;") - logger.debug("create the coprocessing table") - main_db_con.executescript( - """create table coprocessing(coproc_id integer, label text, description text);""") - - logger.debug("finished: main.db cleanup") - - -# =============================================================================== - - -def db_populate_tables(the_scenario, logger): - logger.info("start: db_populate_tables") - - # populate schedules table - populate_schedules_table(the_scenario, logger) - - # populate locations table - populate_locations_table(the_scenario, logger) - - # populate coprocessing table - populate_coprocessing_table(the_scenario, logger) - - # populate the facilities, commodities, and facility_commodities table - # with the input CSVs. - # Note: processor_candidate_commodity_data is generated for FTOT generated candidate - # processors at the end of the candidate generation step. - - for commodity_input_file in [the_scenario.rmp_commodity_data, - the_scenario.destinations_commodity_data, - the_scenario.processors_commodity_data, - the_scenario.processor_candidates_commodity_data]: - # this should just catch processors not specified. - if str(commodity_input_file).lower() == "null" or str(commodity_input_file).lower() == "none": - logger.debug("Commodity Input Data specified in the XML: {}".format(commodity_input_file)) - continue - - else: - populate_facility_commodities_table(the_scenario, commodity_input_file, logger) - - # re issue #109- this is a good place to check if there are multiple input commodities for a processor. - db_check_multiple_input_commodities_for_processor(the_scenario, logger) - - # can delete the tmp_facility_locations table now - db_drop_table(the_scenario, "tmp_facility_locations", logger) - - logger.debug("finished: db_populate_tables") - - -# =================================================================================================== - - -def db_report_commodity_potentials(the_scenario, logger): - logger.info("start: db_report_commodity_potentials") - - # This query pulls the total quantity of each commodity from the facility_commodities table. - # It groups by commodity_name, facility type, and units. The io field is included to help the user - # determine the potential supply, demand, and processing capabilities in the scenario. - - # ----------------------------------- - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc;""" - db_cur = db_con.execute(sql) - - db_data = db_cur.fetchall() - logger.result("-------------------------------------------------------------------") - logger.result("Scenario Total Supply and Demand, and Available Processing Capacity") - logger.result("-------------------------------------------------------------------") - logger.result("note: processor input and outputs are based on facility size and \n reflect a processing " - "capacity, not a conversion of the scenario feedstock supply") - logger.result("commodity_name | facility_type | io | quantity | units ") - logger.result("---------------|---------------|----|---------------|----------") - for row in db_data: - logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], - row[4])) - logger.result("-------------------------------------------------------------------") - # add the ignored processing capacity - # note this doesn't happen until the bx step. - # ------------------------------------------- - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units, f.ignore_facility - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility != 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units, f.ignore_facility - order by commodity_name, io asc;""" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - if len(db_data) > 0: - logger.result("-------------------------------------------------------------------") - logger.result("Scenario Stranded Supply, Demand, and Processing Capacity") - logger.result("-------------------------------------------------------------------") - logger.result("note: stranded supply refers to facilities that are ignored from the analysis.") - logger.result("commodity_name | facility_type | io | quantity | units | ignored ") - logger.result("---------------|---------------|----|---------------|-------|---------") - for row in db_data: - logger.result("{:15.15} {:15.15} {:2.1} {:15,.1f} {:10.10} {:10.10}".format(row[0], row[1], row[2], row[3], - row[4], row[5])) - logger.result("-------------------------------------------------------------------") - - # report out net quantities with ignored facilities removed from the query - # ------------------------------------------------------------------------- - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility == 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc;""" - db_cur = db_con.execute(sql) - - db_data = db_cur.fetchall() - logger.result("-------------------------------------------------------------------") - logger.result("Scenario Net Supply and Demand, and Available Processing Capacity") - logger.result("-------------------------------------------------------------------") - logger.result("note: net supply, demand, and processing capacity ignores facilities not connected to the " - "network.") - logger.result("commodity_name | facility_type | io | quantity | units ") - logger.result("---------------|---------------|----|---------------|----------") - for row in db_data: - logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], - row[4])) - logger.result("-------------------------------------------------------------------") - - -# =================================================================================================== - - -def load_schedules_input_data(schedule_input_file, logger): - - logger.debug("start: load_schedules_input_data") - - import os - if schedule_input_file == "None": - logger.info('schedule file not specified.') - return {'default': {0: 1}} # return dict with global value of default schedule - elif not os.path.exists(schedule_input_file): - logger.warning("warning: cannot find schedule file: {}".format(schedule_input_file)) - return {'default': {0: 1}} - - # create temp dict to store schedule input - schedules = {} - - # read through facility_commodities input CSV - import csv - with open(schedule_input_file, 'rt') as f: - - reader = csv.DictReader(f) - # adding row index for issue #220 to alert user on which row their error is in - for index, row in enumerate(reader): - - schedule_name = str(row['schedule']).lower() # convert schedule to lowercase - day = int(row['day']) # cast day to an int - availability = float(row['availability']) # cast availability to float - - if schedule_name in list(schedules.keys()): - schedules[schedule_name][day] = availability - else: - schedules[schedule_name] = {day: availability} # initialize sub-dict - - # Enforce default schedule req. and default availability req. for all schedules. - # if user has not defined 'default' schedule - if 'default' not in schedules: - logger.debug("Default schedule not found. Adding 'default' with default availability of 1.") - schedules['default'] = {0: 1} - # if schedule does not have a default value (value assigned to day 0), then add as 1. - for schedule_name in list(schedules.keys()): - if 0 not in list(schedules[schedule_name].keys()): - logger.debug("Schedule {} missing default value. Adding default availability of 1.".format(schedule_name)) - schedules[schedule_name][0] = 1 - - return schedules - - -# =================================================================================================== - - -def populate_schedules_table(the_scenario, logger): - - logger.info("start: populate_schedules_table") - - schedules_dict = load_schedules_input_data(the_scenario.schedule, logger) - - # connect to db - with sqlite3.connect(the_scenario.main_db) as db_con: - id_num = 0 - - for schedule_name, schedule_data in iteritems(schedules_dict): - id_num += 1 # 1-index - - # add schedule name into schedule_names table - sql = "insert into schedule_names " \ - "(schedule_id, schedule_name) " \ - "values ({},'{}');".format(id_num, schedule_name) - db_con.execute(sql) - - # add each day into schedules table - for day, availability in iteritems(schedule_data): - sql = "insert into schedules " \ - "(schedule_id, day, availability) " \ - "values ({},{},{});".format(id_num, day, availability) - db_con.execute(sql) - - logger.debug("finished: populate_locations_table") - - -# ============================================================================== - - -def check_for_input_error(input_type, input_val, filename, index, units=None): - """ - :param input_type: a string with the type of input (e.g. 'io', 'facility_name', etc. - :param input_val: a string from the csv with the actual input - :param index: the row index - :param filename: the name of the file containing the row - :param units: string, units used -- only if type == commodity_phase - :return: None if data is valid, or proper error message otherwise - """ - error_message = None - index = index+2 # account for header and 0-indexing (python) conversion to 1-indexing (excel) - if input_type == 'io': - if not (input_val in ['i', 'o']): - error_message = "There is an error in the io entry in row {} of {}. " \ - "Entries should be 'i' or 'o'.".format(index, filename) - elif input_type == 'facility_type': - if not (input_val in ['raw_material_producer', 'processor', 'ultimate_destination']): - error_message = "There is an error in the facility_type entry in row {} of {}. " \ - "The entry is not one of 'raw_material_producer', 'processor', or " \ - "'ultimate_destination'." \ - .format(index, filename) - elif input_type == 'commodity_phase': - # make sure units specified - if units is None: - error_message = "The units in row {} of {} are not specified. Note that solids must have units of mass " \ - "and liquids must have units of volume." \ - .format(index, filename) - elif input_val == 'solid': - # check if units are valid units for solid (dimension of units must be mass) - try: - if not str(ureg(units).dimensionality) == '[mass]': - error_message = "The phase_of_matter entry in row {} of {} is solid, but the units are {}" \ - " which is not a valid unit for this phase of matter. Solids must be measured in " \ - "units of mass." \ - .format(index, filename, units) - except: - error_message = "The phase_of_matter entry in row {} of {} is solid, but the units are {}" \ - " which is not a valid unit for this phase of matter. Solids must be measured in " \ - "units of mass." \ - .format(index, filename, units) - elif input_val == 'liquid': - # check if units are valid units for liquid (dimension of units must be volume, aka length^3) - try: - if not str(ureg(units).dimensionality) == '[length] ** 3': - error_message = "The phase_of_matter entry in row {} of {} is liquid, but the units are {}" \ - " which is not a valid unit for this phase of matter. Liquids must be measured" \ - " in units of volume." \ - .format(index, filename, units) - except: - error_message = "The phase_of_matter entry in row {} of {} is liquid, but the units are {}" \ - " which is not a valid unit for this phase of matter. Liquids must be measured" \ - " in units of volume." \ - .format(index, filename, units) - else: - # throw error that phase is neither solid nor liquid - error_message = "There is an error in the phase_of_matter entry in row {} of {}. " \ - "The entry is not one of 'solid' or 'liquid'." \ - .format(index, filename) - - elif input_type == 'commodity_quantity': - try: - float(input_val) - except ValueError: - error_message = "There is an error in the value entry in row {} of {}. " \ - "The entry is empty or non-numeric (check for extraneous characters)." \ - .format(index, filename) - - return error_message - - -# ============================================================================== - - -def load_facility_commodities_input_data(the_scenario, commodity_input_file, logger): - logger.debug("start: load_facility_commodities_input_data") - if not os.path.exists(commodity_input_file): - logger.warning("warning: cannot find commodity_input file: {}".format(commodity_input_file)) - return - - # create a temp dict to store values from CSV - temp_facility_commodities_dict = {} - - # create empty dictionary to manage schedule input - facility_schedule_dict = {} - - # read through facility_commodities input CSV - import csv - with open(commodity_input_file, 'rt') as f: - - reader = csv.DictReader(f) - # adding row index for issue #220 to alert user on which row their error is in - for index, row in enumerate(reader): - # re: issue #149 -- if the line is empty, just skip it - if list(row.values())[0] == '': - logger.debug('the CSV file has a blank in the first column. Skipping this line: {}'.format( - list(row.values()))) - continue - # {'units': 'kgal', 'facility_name': 'd:01053', 'phase_of_matter': 'liquid', 'value': '9181.521484', - # 'commodity': 'diesel', 'io': 'o', 'share_max_transport_distance'; 'Y'} - io = row["io"] - facility_name = str(row["facility_name"]) - facility_type = row["facility_type"] - commodity_name = row["commodity"].lower() # re: issue #131 - make all commodities lower case - commodity_quantity = row["value"] - commodity_unit = str(row["units"]).replace(' ', '_').lower() # remove spaces and make units lower case - commodity_phase = row["phase_of_matter"] - - # check for proc_cand-specific "non-commodities" to ignore validation (issue #254) - non_commodities = ['minsize', 'maxsize', 'cost_formula', 'min_aggregation'] - - # input data validation - if commodity_name not in non_commodities: # re: issue #254 only test actual commodities - # test io - io = io.lower() # convert 'I' and 'O' to 'i' and 'o' - error_message = check_for_input_error("io", io, commodity_input_file, index) - if error_message: - raise Exception(error_message) - # test facility type - error_message = check_for_input_error("facility_type", facility_type, commodity_input_file, index) - if error_message: - raise Exception(error_message) - # test commodity quantity - error_message = check_for_input_error("commodity_quantity", commodity_quantity, commodity_input_file, index) - if error_message: - raise Exception(error_message) - # test commodity phase - error_message = check_for_input_error("commodity_phase", commodity_phase, commodity_input_file, index, - units=commodity_unit) - if error_message: - raise Exception(error_message) - else: - logger.debug("Skipping input validation on special candidate processor commodity: {}" - .format(commodity_name)) - - if "max_processor_input" in list(row.keys()): - max_processor_input = row["max_processor_input"] - else: - max_processor_input = "Null" - - if "max_transport_distance" in list(row.keys()): - commodity_max_transport_distance = row["max_transport_distance"] - else: - commodity_max_transport_distance = "Null" - - if "share_max_transport_distance" in list(row.keys()): - share_max_transport_distance = row["share_max_transport_distance"] - else: - share_max_transport_distance = 'N' - - # add schedule_id, if available - if "schedule" in list(row.keys()): - schedule_name = str(row["schedule"]).lower() - - # blank schedule name should be cast to default - if schedule_name == "none": - schedule_name = "default" - else: - schedule_name = "default" - - # manage facility_schedule_dict - if facility_name not in facility_schedule_dict: - facility_schedule_dict[facility_name] = schedule_name - elif facility_schedule_dict[facility_name] != schedule_name: - logger.info("Schedule name '{}' does not match previously entered schedule '{}' for facility '{}'". - format(schedule_name, facility_schedule_dict[facility_name], facility_name)) - schedule_name = facility_schedule_dict[facility_name] - - # use pint to set the quantity and units - commodity_quantity_and_units = Q_(float(commodity_quantity), commodity_unit) - if max_processor_input != 'Null': - max_input_quantity_and_units = Q_(float(max_processor_input), commodity_unit) - - if commodity_phase.lower() == 'liquid': - commodity_unit = the_scenario.default_units_liquid_phase - if commodity_phase.lower() == 'solid': - commodity_unit = the_scenario.default_units_solid_phase - - if commodity_name == 'cost_formula': - pass - else: - commodity_quantity = commodity_quantity_and_units.to(commodity_unit).magnitude - - if max_processor_input != 'Null': - max_processor_input = max_input_quantity_and_units.to(commodity_unit).magnitude - - # add to the dictionary of facility_commodities mapping - if facility_name not in list(temp_facility_commodities_dict.keys()): - temp_facility_commodities_dict[facility_name] = [] - - temp_facility_commodities_dict[facility_name].append([facility_type, commodity_name, commodity_quantity, - commodity_unit, commodity_phase, - commodity_max_transport_distance, io, - share_max_transport_distance, max_processor_input, - schedule_name]) - - logger.debug("finished: load_facility_commodities_input_data") - return temp_facility_commodities_dict - - -# ============================================================================== - - -def populate_facility_commodities_table(the_scenario, commodity_input_file, logger): - - logger.debug("start: populate_facility_commodities_table {}".format(commodity_input_file)) - - if not os.path.exists(commodity_input_file): - logger.debug("note: cannot find commodity_input file: {}".format(commodity_input_file)) - return - - facility_commodities_dict = load_facility_commodities_input_data(the_scenario, commodity_input_file, logger) - - candidate = 0 - if os.path.split(commodity_input_file)[1].find("ftot_generated_processor_candidates") > -1: - candidate = 1 - - # connect to main.db and add values to table - # --------------------------------------------------------- - with sqlite3.connect(the_scenario.main_db) as db_con: - for facility_name, facility_data in iteritems(facility_commodities_dict): - - # unpack the facility_type (should be the same for all entries) - facility_type = facility_data[0][0] - facility_type_id = get_facility_id_type(the_scenario, db_con, facility_type, logger) - - location_id = get_facility_location_id(the_scenario, db_con, facility_name, logger) - - # get schedule id from the db - schedule_name = facility_data[0][-1] - schedule_id = get_schedule_id(the_scenario, db_con, schedule_name, logger) - - max_processor_input = facility_data[0][-2] - - # get the facility_id from the db (add the facility if it doesn't exists) - # and set up entry in facility_id table - facility_id = get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger) - - # iterate through each commodity - for commodity_data in facility_data: - - # get commodity_id. (adds commodity if it doesn't exist) - commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger) - - [facility_type, commodity_name, commodity_quantity, commodity_units, commodity_phase, commodity_max_transport_distance, io, share_max_transport_distance, unused_var_max_processor_input, schedule_id] = commodity_data - - if not commodity_quantity == "0.0": # skip anything with no material - sql = "insert into facility_commodities " \ - "(facility_id, location_id, commodity_id, quantity, units, io, share_max_transport_distance) " \ - "values ('{}','{}', '{}', '{}', '{}', '{}', '{}');".format( - facility_id, location_id, commodity_id, commodity_quantity, commodity_units, io, share_max_transport_distance) - db_con.execute(sql) - else: - logger.debug("skipping commodity_data {} because quantity: {}".format(commodity_name, commodity_quantity)) - db_con.execute("""update commodities - set share_max_transport_distance = - (select 'Y' from facility_commodities fc - where commodities.commodity_id = fc.commodity_id - and fc.share_max_transport_distance = 'Y') - where exists (select 'Y' from facility_commodities fc - where commodities.commodity_id = fc.commodity_id - and fc.share_max_transport_distance = 'Y') - ;""" - ) - - logger.debug("finished: populate_facility_commodities_table") - - -# ============================================================================== - - -def db_check_multiple_input_commodities_for_processor(the_scenario, logger): - # connect to main.db and add values to table - # --------------------------------------------------------- - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select f.facility_name, count(*) - from facility_commodities fc - join facilities f on f.facility_ID = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where fti.facility_type like 'processor' and fc.io like 'i' - group by f.facility_name - having count(*) > 1;""" - db_cur = db_con.execute(sql) - data = db_cur.fetchall() - if len(data) > 0: - for multi_input_processor in data: - logger.warning("Processor: {} has {} input commodities specified.".format(multi_input_processor[0], - multi_input_processor[1])) - logger.warning("Multiple processor inputs are not supported in the same scenario as shared max transport " - "distance") - # logger.info("make adjustments to the processor commodity input file: {}".format(the_scenario.processors_commodity_data)) - # error = "Multiple input commodities for processors is not supported in FTOT" - # logger.error(error) - # raise Exception(error) - - -# ============================================================================== - - -def populate_coprocessing_table(the_scenario, logger): - - logger.info("start: populate_coprocessing_table") - - # connect to db - with sqlite3.connect(the_scenario.main_db) as db_con: - - # should be filled from the file coprocessing.csv, which needs to be added to xml still - # I would place this file in the common data folder probably, since it is a fixed reference table - # for now, filling the db table here manually with the data from the csv file - sql = """INSERT INTO coprocessing (coproc_id, label, description) - VALUES - (1, 'single', 'code should throw error if processor has more than one input commodity'), - (2, 'fixed combination', 'every input commodity listed for the processor is required, in the ratio ' || - 'specified by their quantities. Output requires all inputs to be present; '), - (3, 'substitutes allowed', 'any one of the input commodities listed for the processor can be used ' || - 'to generate the output, with the ratios specified by quantities. A ' || - 'combination of inputs is allowed. '), - (4, 'external input', 'assumes all specified inputs are required, in that ratio, with the addition ' || - 'of an External or Non-Transported input commodity. This is included in the ' || - 'ratio and as part of the input capacity, but is available in unlimited ' || - 'quantity at the processor location. ') - ; """ - db_con.execute(sql) - - logger.debug("not yet implemented: populate_coprocessing_table") - - -# ============================================================================= - - -def populate_locations_table(the_scenario, logger): - - logger.info("start: populate_locations_table") - - # connect to db - with sqlite3.connect(the_scenario.main_db) as db_con: - - # then iterate through the GIS and get the facility_name, shape_x and shape_y - with arcpy.da.Editor(the_scenario.main_gdb) as edit: - - logger.debug("iterating through the FC and create a facility_location mapping with x,y coord.") - - for fc in [the_scenario.rmp_fc, the_scenario.destinations_fc, the_scenario.processors_fc]: - - logger.debug("iterating through the FC: {}".format(fc)) - with arcpy.da.SearchCursor(fc, ["facility_name", "SHAPE@X", "SHAPE@Y"]) as cursor: - for row in cursor: - facility_name = row[0] - shape_x = round(row[1], -2) - shape_y = round(row[2], -2) - - # check if location_id exists exists for snap_x and snap_y - location_id = get_location_id(the_scenario, db_con, shape_x, shape_y, logger) - - if location_id > 0: - # map facility_name to the location_id in the tmp_facility_locations table - db_con.execute("insert or ignore into tmp_facility_locations " - "(facility_name, location_id) " - "values ('{}', '{}');".format(facility_name, location_id)) - else: - error = "no location_id exists for shape_x {} and shape_y {}".format(shape_x, shape_y) - logger.error(error) - raise Exception(error) - - logger.debug("finished: populate_locations_table") - - -# ============================================================================= - - -def get_location_id(the_scenario, db_con, shape_x, shape_y, logger): - - location_id = None - - # get location_id - for row in db_con.execute(""" - select - location_id - from locations l - where l.shape_x = '{}' and l.shape_y = '{}';""".format(shape_x, shape_y)): - - location_id = row[0] - - # if no ID, add it. - if location_id is None: - # if it doesn't exist, add to locations table and generate a location id. - db_cur = db_con.execute("insert into locations (shape_x, shape_y) values ('{}', '{}');".format(shape_x, shape_y)) - location_id = db_cur.lastrowid - - # check again. we should have the ID now. if we don't throw an error. - if location_id is None: - error = "something went wrong getting location_id shape_x: {}, shape_y: {} location_id ".format(shape_x, shape_y) - logger.error(error) - raise Exception(error) - else: - return location_id - - -# ============================================================================= - - -def get_facility_location_id(the_scenario, db_con, facility_name, logger): - - # get location_id - db_cur = db_con.execute("select location_id from tmp_facility_locations l where l.facility_name = '{}';".format(str(facility_name))) - location_id = db_cur.fetchone() - if not location_id: - warning = "location_id for tmp_facility_name: {} is not found.".format(facility_name) - logger.debug(warning) - else: - return location_id[0] - - -# ============================================================================= - - -def get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger): - - # if it doesn't exist, add to facilities table and generate a facility id. - db_con.execute("insert or ignore into facilities " - "(location_id, facility_name, facility_type_id, candidate, schedule_id, max_capacity) " - "values ('{}', '{}', {}, {}, {}, {});".format(location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input)) - - # get facility_id - db_cur = db_con.execute("select facility_id " - "from facilities f " - "where f.facility_name = '{}' and facility_type_id = {};".format(facility_name, facility_type_id)) - facility_id = db_cur.fetchone()[0] - - if not facility_id: - error = "something went wrong getting {} facility_id ".format(facility_id) - logger.error(error) - raise Exception(error) - else: - return facility_id - - -# =================================================================================================== - - -def get_facility_id_type(the_scenario, db_con, facility_type, logger): - facility_type_id = None - - # get facility_id_type - for row in db_con.execute("select facility_type_id " - "from facility_type_id f " - "where facility_type = '{}';".format(facility_type)): - facility_type_id = row[0] - - # if it doesn't exist, add to facility_type table and generate a facility_type_id. - if facility_type_id is None: - db_cur = db_con.execute("insert into facility_type_id (facility_type) values ('{}');".format(facility_type)) - facility_type_id = db_cur.lastrowid - - # check again if we have facility_type_id - if facility_type_id is None: - error = "something went wrong getting {} facility_type_id ".format(facility_type) - logger.error(error) - raise Exception(error) - else: - return facility_type_id - - -# =================================================================================================== - - -def get_commodity_id(the_scenario, db_con, commodity_data, logger): - - [facility_type, commodity_name, commodity_quantity, commodity_unit, commodity_phase, - commodity_max_transport_distance, io, share_max_transport_distance, max_processor_input, schedule_id] = commodity_data - - # get the commodity_id. - db_cur = db_con.execute("select commodity_id " - "from commodities c " - "where c.commodity_name = '{}';".format(commodity_name)) - - commodity_id = db_cur.fetchone() # don't index the list, since it might not exists. - - if not commodity_id: - # if it doesn't exist, add the commodity to the commodities table and generate a commodity id - if commodity_max_transport_distance in ['Null', '', 'None']: - sql = "insert into commodities " \ - "(commodity_name, units, phase_of_matter, share_max_transport_distance) " \ - "values ('{}', '{}', '{}','{}');".format(commodity_name, commodity_unit, commodity_phase, - share_max_transport_distance) - else: - sql = "insert into commodities " \ - "(commodity_name, units, phase_of_matter, max_transport_distance, share_max_transport_distance) " \ - "values ('{}', '{}', '{}', {}, '{}');".format(commodity_name, commodity_unit, commodity_phase, - commodity_max_transport_distance, - share_max_transport_distance) - db_con.execute(sql) - - # get the commodity_id. - db_cur = db_con.execute("select commodity_id " - "from commodities c " - "where c.commodity_name = '{}';".format(commodity_name)) - commodity_id = db_cur.fetchone()[0] # index the first (and only) data in the list - - if not commodity_id: - error = "something went wrong adding the commodity {} " \ - "to the commodities table and getting a commodity_id".format(commodity_name) - logger.error(error) - raise Exception(error) - else: - return commodity_id - - -# =================================================================================================== - - -def get_schedule_id(the_scenario, db_con, schedule_name, logger): - # get location_id - db_cur = db_con.execute( - "select schedule_id from schedule_names s where s.schedule_name = '{}';".format(str(schedule_name))) - schedule_id = db_cur.fetchone() - if not schedule_id: - # if schedule id is not found, replace with the default schedule - warning = 'schedule_id for schedule_name: {} is not found. Replace with default'.format(schedule_name) - logger.info(warning) - db_cur = db_con.execute( - "select schedule_id from schedule_names s where s.schedule_name = 'default';") - schedule_id = db_cur.fetchone() - - return schedule_id[0] - - -# =================================================================================================== - - -def gis_clean_fc(the_scenario, logger): - - logger.info("start: gis_clean_fc") - - start_time = datetime.datetime.now() - - # clear the destinations - gis_clear_feature_class(the_scenario.destinations_fc, logger) - - # clear the RMPs - gis_clear_feature_class(the_scenario.rmp_fc, logger) - - # clear the processors - gis_clear_feature_class(the_scenario.processors_fc, logger) - - # clear the processors - gis_clear_feature_class(the_scenario.locations_fc, logger) - - logger.debug("finished: gis_clean_fc: Runtime (HMS): \t{}".format - (ftot_supporting.get_total_runtime_string(start_time))) - - -# ============================================================================== - - -def gis_clear_feature_class(fc, logger): - logger.debug("start: clear_feature_class for fc {}".format(os.path.split(fc)[1])) - if arcpy.Exists(fc): - arcpy.Delete_management(fc) - logger.debug("finished: deleted existing fc {}".format(os.path.split(fc)[1])) - - -# =================================================================================================== - - -def gis_get_feature_count(fc, logger): - result = arcpy.GetCount_management(fc) - count = int(result.getOutput(0)) - return count - - -# =================================================================================================== - - -def gis_populate_fc(the_scenario, logger): - - logger.info("start: gis_populate_fc") - - start_time = datetime.datetime.now() - - # populate the destinations fc in main.gdb - gis_ultimate_destinations_setup_fc(the_scenario, logger) - - # populate the RMPs fc in main.gdb - gis_rmp_setup_fc(the_scenario, logger) - - # populate the processors fc in main.gdb - gis_processors_setup_fc(the_scenario, logger) - - logger.debug("finished: gis_populate_fc: Runtime (HMS): \t{}".format - (ftot_supporting.get_total_runtime_string(start_time))) - - -# ------------------------------------------------------------ - - -def gis_ultimate_destinations_setup_fc(the_scenario, logger): - - logger.info("start: gis_ultimate_destinations_setup_fc") - - start_time = datetime.datetime.now() - - # copy the destination from the baseline layer to the scenario gdb - # -------------------------------------------------------------- - if not arcpy.Exists(the_scenario.base_destination_layer): - error = "can't find baseline data destinations layer {}".format(the_scenario.base_destination_layer) - raise IOError(error) - - destinations_fc = the_scenario.destinations_fc - arcpy.Project_management(the_scenario.base_destination_layer, destinations_fc, ftot_supporting_gis.LCC_PROJ) - - # Delete features with no data in csv -- cleans up GIS output and eliminates unnecessary GIS processing - # -------------------------------------------------------------- - # create a temp dict to store values from CSV - temp_facility_commodities_dict = {} - counter = 0 - - # read through facility_commodities input CSV - import csv - with open(the_scenario.destinations_commodity_data, 'rt') as f: - reader = csv.DictReader(f) - for row in reader: - facility_name = str(row["facility_name"]) - commodity_quantity = float(row["value"]) - - if facility_name not in list(temp_facility_commodities_dict.keys()): - if commodity_quantity > 0: - temp_facility_commodities_dict[facility_name] = True - - with arcpy.da.UpdateCursor(destinations_fc, ['Facility_Name']) as cursor: - for row in cursor: - if row[0] in temp_facility_commodities_dict: - pass - else: - cursor.deleteRow() - counter += 1 - del cursor - logger.config("Number of Destinations removed due to lack of commodity data: \t{}".format(counter)) - - with arcpy.da.SearchCursor(destinations_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: - for row in scursor: - # Check if coordinates of facility are roughly within North America - if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: - pass - else: - logger.warning("Facility: {} is not located in North America.".format(row[0])) - logger.info("remove the facility from the scenario or make adjustments to the facility's location in" - " the destinations feature class: {}".format(the_scenario.base_destination_layer)) - error = "Facilities outside North America are not supported in FTOT" - logger.error(error) - raise Exception(error) - - result = gis_get_feature_count(destinations_fc, logger) - logger.config("Number of Destinations: \t{}".format(result)) - - logger.debug("finish: gis_ultimate_destinations_setup_fc: Runtime (HMS): \t{}".format - (ftot_supporting.get_total_runtime_string(start_time))) - - -# ============================================================================= - - -def gis_rmp_setup_fc(the_scenario, logger): - - logger.info("start: gis_rmp_setup_fc") - start_time = datetime.datetime.now() - - # copy the rmp from the baseline data to the working gdb - # ---------------------------------------------------------------- - if not arcpy.Exists(the_scenario.base_rmp_layer): - error = "can't find baseline data rmp layer {}".format(the_scenario.base_rmp_layer) - raise IOError(error) - - rmp_fc = the_scenario.rmp_fc - arcpy.Project_management(the_scenario.base_rmp_layer, rmp_fc, ftot_supporting_gis.LCC_PROJ) - - # Delete features with no data in csv-- cleans up GIS output and eliminates unnecessary GIS processing - # -------------------------------------------------------------- - # create a temp dict to store values from CSV - temp_facility_commodities_dict = {} - counter = 0 - - # read through facility_commodities input CSV - import csv - with open(the_scenario.rmp_commodity_data, 'rt') as f: - - reader = csv.DictReader(f) - for row in reader: - facility_name = str(row["facility_name"]) - commodity_quantity = float(row["value"]) - - if facility_name not in list(temp_facility_commodities_dict.keys()): - if commodity_quantity > 0: - temp_facility_commodities_dict[facility_name] = True - - with arcpy.da.UpdateCursor(rmp_fc, ['Facility_Name']) as cursor: - for row in cursor: - if row[0] in temp_facility_commodities_dict: - pass - else: - cursor.deleteRow() - counter +=1 - del cursor - logger.config("Number of RMPs removed due to lack of commodity data: \t{}".format(counter)) - - with arcpy.da.SearchCursor(rmp_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: - for row in scursor: - # Check if coordinates of facility are roughly within North America - if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: - pass - else: - logger.warning("Facility: {} is not located in North America.".format(row[0])) - logger.info("remove the facility from the scenario or make adjustments to the facility's location in " - "the RMP feature class: {}".format(the_scenario.base_rmp_layer)) - error = "Facilities outside North America are not supported in FTOT" - logger.error(error) - raise Exception(error) - - del scursor - - result = gis_get_feature_count(rmp_fc, logger) - logger.config("Number of RMPs: \t{}".format(result)) - - logger.debug("finished: gis_rmp_setup_fc: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - -# ============================================================================= - - -def gis_processors_setup_fc(the_scenario, logger): - - logger.info("start: gis_processors_setup_fc") - start_time = datetime.datetime.now() - - if str(the_scenario.base_processors_layer).lower() == "null" or \ - str(the_scenario.base_processors_layer).lower() == "none": - # create an empty processors layer - # ------------------------- - processors_fc = the_scenario.processors_fc - - if arcpy.Exists(processors_fc): - arcpy.Delete_management(processors_fc) - logger.debug("deleted existing {} layer".format(processors_fc)) - - arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "processors", "POINT", "#", "DISABLED", "DISABLED", - ftot_supporting_gis.LCC_PROJ, "#", "0", "0", "0") - - arcpy.AddField_management(processors_fc, "Facility_Name", "TEXT", "#", "#", "25", "#", "NULLABLE", - "NON_REQUIRED", "#") - arcpy.AddField_management(processors_fc, "Candidate", "SHORT") - - else: - # copy the processors from the baseline data to the working gdb - # ---------------------------------------------------------------- - if not arcpy.Exists(the_scenario.base_processors_layer): - error = "can't find baseline data processors layer {}".format(the_scenario.base_processors_layer) - raise IOError(error) - - processors_fc = the_scenario.processors_fc - arcpy.Project_management(the_scenario.base_processors_layer, processors_fc, ftot_supporting_gis.LCC_PROJ) - - arcpy.AddField_management(processors_fc, "Candidate", "SHORT") - - # Delete features with no data in csv-- cleans up GIS output and eliminates unnecessary GIS processing - # -------------------------------------------------------------- - # create a temp dict to store values from CSV - temp_facility_commodities_dict = {} - counter = 0 - - # read through facility_commodities input CSV - import csv - with open(the_scenario.processors_commodity_data, 'rt') as f: - - reader = csv.DictReader(f) - for row in reader: - facility_name = str(row["facility_name"]) - commodity_quantity = float(row["value"]) - - if facility_name not in list(temp_facility_commodities_dict.keys()): - if commodity_quantity > 0: - temp_facility_commodities_dict[facility_name] = True - - with arcpy.da.UpdateCursor(processors_fc, ['Facility_Name']) as cursor: - for row in cursor: - if row[0] in temp_facility_commodities_dict: - pass - else: - cursor.deleteRow() - counter += 1 - - del cursor - logger.config("Number of processors removed due to lack of commodity data: \t{}".format(counter)) - - with arcpy.da.SearchCursor(processors_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: - for row in scursor: - # Check if coordinates of facility are roughly within North America - if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: - pass - else: - logger.warning("Facility: {} is not located in North America.".format(row[0])) - logger.info("remove the facility from the scenario or make adjustments to the facility's location " - "in the processors feature class: {}".format(the_scenario.base_processors_layer)) - error = "Facilities outside North America are not supported in FTOT" - logger.error(error) - raise Exception(error) - - del scursor - - # check for candidates or other processors specified in either XML or - layers_to_merge = [] - - # add the candidates_for_merging if they exists. - if arcpy.Exists(the_scenario.processor_candidates_fc): - logger.info("adding {} candidate processors to the processors fc".format( - gis_get_feature_count(the_scenario.processor_candidates_fc, logger))) - layers_to_merge.append(the_scenario.processor_candidates_fc) - gis_merge_processor_fc(the_scenario, layers_to_merge, logger) - - result = gis_get_feature_count(processors_fc, logger) - - logger.config("Number of Processors: \t{}".format(result)) - - logger.debug("finish: gis_processors_setup_fc: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - -# ======================================================================= - - -def gis_merge_processor_fc(the_scenario, layers_to_merge, logger): - - logger.info("merge the candidates and processors together. ") - - scenario_gdb = the_scenario.main_gdb - - # ------------------------------------------------------------------------- - # append the processors from the kayers to merge data to the working gdb - # ------------------------------------------------------------------------- - if len(layers_to_merge) == 0: - error = "len of layers_to_merge for processors is {}".format(layers_to_merge) - raise IOError(error) - - processors_fc = the_scenario.processors_fc - - # Open an edit session and start an edit operation - with arcpy.da.Editor(scenario_gdb) as edit: - # Report out some information about how many processors are in each layer to merge - for layer in layers_to_merge: - - processor_count = str(arcpy.GetCount_management(layer)) - logger.debug("Importing {} processors from {}".format(processor_count, os.path.split(layer)[0])) - logger.info("Append candidates into the Processors FC") - - arcpy.Append_management(layer, processors_fc, "NO_TEST") - - total_processor_count = str(arcpy.GetCount_management(processors_fc)) - logger.debug("Total count: {} records in {}".format(total_processor_count, os.path.split(processors_fc)[0])) - - return - +# --------------------------------------------------------------------------------------------------- +# Name: ftot_facilities +# +# Purpose: setup for raw material producers (RMP) and ultimate destinations. +# adds demand for destinations. adds supply for RMP. +# hooks facilities into the network using artificial links. +# special consideration for how pipeline links are added. +# +# --------------------------------------------------------------------------------------------------- + +import ftot_supporting +import ftot_supporting_gis +import arcpy +import datetime +import os +import sqlite3 +from ftot import ureg, Q_ +from six import iteritems +LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') + + +# =============================================================================== + + +def facilities(the_scenario, logger): + gis_clean_fc(the_scenario, logger) + gis_populate_fc(the_scenario, logger) + + db_cleanup_tables(the_scenario, logger) + db_populate_tables(the_scenario, logger) + db_report_commodity_potentials(the_scenario, logger) + + if the_scenario.processors_candidate_slate_data != 'None': + # make candidate_process_list and candidate_process_commodities tables + from ftot_processor import generate_candidate_processor_tables + generate_candidate_processor_tables(the_scenario, logger) + + +# =============================================================================== + + +def db_drop_table(the_scenario, table_name, logger): + with sqlite3.connect(the_scenario.main_db) as main_db_con: + logger.debug("drop the {} table".format(table_name)) + main_db_con.execute("drop table if exists {};".format(table_name)) + + +# ============================================================================== + + +def db_cleanup_tables(the_scenario, logger): + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + # DB CLEAN UP + # ------------ + logger.info("start: db_cleanup_tables") + + # a new run is a new scenario in the main.db + # so drop and create the following tables if they exists + # -------------------------------------------- + + # locations table + logger.debug("drop the locations table") + main_db_con.execute("drop table if exists locations;") + logger.debug("create the locations table") + main_db_con.executescript( + "create table locations(location_ID INTEGER PRIMARY KEY, shape_x real, shape_y real, ignore_location text);") + + # tmp_facility_locations table + # a temp table used to map the facility_name from the facility_commodities data csv + # to the location_id. its used to populate the facility_commodities table + # and deleted after it is populated. + logger.debug("drop the tmp_facility_locations table") + main_db_con.execute("drop table if exists tmp_facility_locations;") + logger.debug("create the tmp_facility_locations table") + main_db_con.executescript( + "create table tmp_facility_locations(location_ID INTEGER , facility_name text PRIMARY KEY);") + + # facilities table + logger.debug("drop the facilities table") + main_db_con.execute("drop table if exists facilities;") + logger.debug("create the facilities table") + main_db_con.executescript( + "create table facilities(facility_ID INTEGER PRIMARY KEY, location_id integer, facility_name text, facility_type_id integer, ignore_facility text, candidate binary, schedule_id integer, max_capacity float);") + + # facility_type_id table + logger.debug("drop the facility_type_id table") + main_db_con.execute("drop table if exists facility_type_id;") + main_db_con.executescript("create table facility_type_id(facility_type_id integer primary key, facility_type text);") + + # phase_of_matter_id table + logger.debug("drop the phase_of_matter_id table") + main_db_con.execute("drop table if exists phase_of_matter_id;") + logger.debug("create the phase_of_matter_id table") + main_db_con.executescript( + "create table phase_of_matter_id(phase_of_matter_id INTEGER PRIMARY KEY, phase_of_matter text);") + + # facility_commodities table + # tracks the relationship of facility and commodities in or out + logger.debug("drop the facility_commodities table") + main_db_con.execute("drop table if exists facility_commodities;") + logger.debug("create the facility_commodities table") + main_db_con.executescript( + "create table facility_commodities(facility_id integer, location_id integer, commodity_id interger, " + "quantity numeric, units text, io text, share_max_transport_distance text);") + + # commodities table + logger.debug("drop the commodities table") + main_db_con.execute("drop table if exists commodities;") + logger.debug("create the commodities table") + main_db_con.executescript( + """create table commodities(commodity_ID INTEGER PRIMARY KEY, commodity_name text, supertype text, subtype text, + units text, phase_of_matter text, max_transport_distance numeric, proportion_of_supertype numeric, + share_max_transport_distance text, CONSTRAINT unique_name UNIQUE(commodity_name) );""") + # proportion_of_supertype specifies how much demand is satisfied by this subtype relative to the "pure" + # fuel/commodity. this will depend on the process + + # schedule_names table + logger.debug("drop the schedule names table") + main_db_con.execute("drop table if exists schedule_names;") + logger.debug("create the schedule names table") + main_db_con.executescript( + """create table schedule_names(schedule_id INTEGER PRIMARY KEY, schedule_name text);""") + + # schedules table + logger.debug("drop the schedules table") + main_db_con.execute("drop table if exists schedules;") + logger.debug("create the schedules table") + main_db_con.executescript( + """create table schedules(schedule_id integer, day integer, availability numeric);""") + + # coprocessing reference table + logger.debug("drop the coprocessing table") + main_db_con.execute("drop table if exists coprocessing;") + logger.debug("create the coprocessing table") + main_db_con.executescript( + """create table coprocessing(coproc_id integer, label text, description text);""") + + logger.debug("finished: main.db cleanup") + + +# =============================================================================== + + +def db_populate_tables(the_scenario, logger): + logger.info("start: db_populate_tables") + + # populate schedules table + populate_schedules_table(the_scenario, logger) + + # populate locations table + populate_locations_table(the_scenario, logger) + + # populate coprocessing table + populate_coprocessing_table(the_scenario, logger) + + # populate the facilities, commodities, and facility_commodities table + # with the input CSVs. + # Note: processor_candidate_commodity_data is generated for FTOT generated candidate + # processors at the end of the candidate generation step. + + for commodity_input_file in [the_scenario.rmp_commodity_data, + the_scenario.destinations_commodity_data, + the_scenario.processors_commodity_data, + the_scenario.processor_candidates_commodity_data]: + # this should just catch processors not specified. + if str(commodity_input_file).lower() == "null" or str(commodity_input_file).lower() == "none": + logger.debug("Commodity Input Data specified in the XML: {}".format(commodity_input_file)) + continue + + else: + populate_facility_commodities_table(the_scenario, commodity_input_file, logger) + + # re issue #109- this is a good place to check if there are multiple input commodities for a processor. + db_check_multiple_input_commodities_for_processor(the_scenario, logger) + + # can delete the tmp_facility_locations table now + db_drop_table(the_scenario, "tmp_facility_locations", logger) + + logger.debug("finished: db_populate_tables") + + +# =================================================================================================== + + +def db_report_commodity_potentials(the_scenario, logger): + logger.info("start: db_report_commodity_potentials") + + # This query pulls the total quantity of each commodity from the facility_commodities table. + # It groups by commodity_name, facility type, and units. The io field is included to help the user + # determine the potential supply, demand, and processing capabilities in the scenario. + + # ----------------------------------- + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc;""" + db_cur = db_con.execute(sql) + + db_data = db_cur.fetchall() + logger.result("-------------------------------------------------------------------") + logger.result("Scenario Total Supply and Demand, and Available Processing Capacity") + logger.result("-------------------------------------------------------------------") + logger.result("note: processor input and outputs are based on facility size and \n reflect a processing " + "capacity, not a conversion of the scenario feedstock supply") + logger.result("commodity_name | facility_type | io | quantity | units ") + logger.result("---------------|---------------|----|---------------|----------") + for row in db_data: + logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], + row[4])) + logger.result("-------------------------------------------------------------------") + # add the ignored processing capacity + # note this doesn't happen until the bx step. + # ------------------------------------------- + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units, f.ignore_facility + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility != 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units, f.ignore_facility + order by commodity_name, io asc;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + if len(db_data) > 0: + logger.result("-------------------------------------------------------------------") + logger.result("Scenario Stranded Supply, Demand, and Processing Capacity") + logger.result("-------------------------------------------------------------------") + logger.result("note: stranded supply refers to facilities that are ignored from the analysis.") + logger.result("commodity_name | facility_type | io | quantity | units | ignored ") + logger.result("---------------|---------------|----|---------------|-------|---------") + for row in db_data: + logger.result("{:15.15} {:15.15} {:2.1} {:15,.1f} {:10.10} {:10.10}".format(row[0], row[1], row[2], row[3], + row[4], row[5])) + logger.result("-------------------------------------------------------------------") + + # report out net quantities with ignored facilities removed from the query + # ------------------------------------------------------------------------- + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility == 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc;""" + db_cur = db_con.execute(sql) + + db_data = db_cur.fetchall() + logger.result("-------------------------------------------------------------------") + logger.result("Scenario Net Supply and Demand, and Available Processing Capacity") + logger.result("-------------------------------------------------------------------") + logger.result("note: net supply, demand, and processing capacity ignores facilities not connected to the " + "network.") + logger.result("commodity_name | facility_type | io | quantity | units ") + logger.result("---------------|---------------|----|---------------|----------") + for row in db_data: + logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], + row[4])) + logger.result("-------------------------------------------------------------------") + + +# =================================================================================================== + + +def load_schedules_input_data(schedule_input_file, logger): + + logger.debug("start: load_schedules_input_data") + + import os + if schedule_input_file == "None": + logger.info('schedule file not specified.') + return {'default': {0: 1}} # return dict with global value of default schedule + elif not os.path.exists(schedule_input_file): + logger.warning("warning: cannot find schedule file: {}".format(schedule_input_file)) + return {'default': {0: 1}} + + # create temp dict to store schedule input + schedules = {} + + # read through facility_commodities input CSV + import csv + with open(schedule_input_file, 'rt') as f: + + reader = csv.DictReader(f) + # adding row index for issue #220 to alert user on which row their error is in + for index, row in enumerate(reader): + + schedule_name = str(row['schedule']).lower() # convert schedule to lowercase + day = int(row['day']) # cast day to an int + availability = float(row['availability']) # cast availability to float + + if schedule_name in list(schedules.keys()): + schedules[schedule_name][day] = availability + else: + schedules[schedule_name] = {day: availability} # initialize sub-dict + + # Enforce default schedule req. and default availability req. for all schedules. + # if user has not defined 'default' schedule + if 'default' not in schedules: + logger.debug("Default schedule not found. Adding 'default' with default availability of 1.") + schedules['default'] = {0: 1} + # if schedule does not have a default value (value assigned to day 0), then add as 1. + for schedule_name in list(schedules.keys()): + if 0 not in list(schedules[schedule_name].keys()): + logger.debug("Schedule {} missing default value. Adding default availability of 1.".format(schedule_name)) + schedules[schedule_name][0] = 1 + + return schedules + + +# =================================================================================================== + + +def populate_schedules_table(the_scenario, logger): + + logger.info("start: populate_schedules_table") + + schedules_dict = load_schedules_input_data(the_scenario.schedule, logger) + + # connect to db + with sqlite3.connect(the_scenario.main_db) as db_con: + id_num = 0 + + for schedule_name, schedule_data in iteritems(schedules_dict): + id_num += 1 # 1-index + + # add schedule name into schedule_names table + sql = "insert into schedule_names " \ + "(schedule_id, schedule_name) " \ + "values ({},'{}');".format(id_num, schedule_name) + db_con.execute(sql) + + # add each day into schedules table + for day, availability in iteritems(schedule_data): + sql = "insert into schedules " \ + "(schedule_id, day, availability) " \ + "values ({},{},{});".format(id_num, day, availability) + db_con.execute(sql) + + logger.debug("finished: populate_locations_table") + + +# ============================================================================== + + +def check_for_input_error(input_type, input_val, filename, index, units=None): + """ + :param input_type: a string with the type of input (e.g. 'io', 'facility_name', etc. + :param input_val: a string from the csv with the actual input + :param index: the row index + :param filename: the name of the file containing the row + :param units: string, units used -- only if type == commodity_phase + :return: None if data is valid, or proper error message otherwise + """ + error_message = None + index = index+2 # account for header and 0-indexing (python) conversion to 1-indexing (excel) + if input_type == 'io': + if not (input_val in ['i', 'o']): + error_message = "There is an error in the io entry in row {} of {}. " \ + "Entries should be 'i' or 'o'.".format(index, filename) + elif input_type == 'facility_type': + if not (input_val in ['raw_material_producer', 'processor', 'ultimate_destination']): + error_message = "There is an error in the facility_type entry in row {} of {}. " \ + "The entry is not one of 'raw_material_producer', 'processor', or " \ + "'ultimate_destination'." \ + .format(index, filename) + elif input_type == 'commodity_phase': + # make sure units specified + if units is None: + error_message = "The units in row {} of {} are not specified. Note that solids must have units of mass " \ + "and liquids must have units of volume." \ + .format(index, filename) + elif input_val == 'solid': + # check if units are valid units for solid (dimension of units must be mass) + try: + if not str(ureg(units).dimensionality) == '[mass]': + error_message = "The phase_of_matter entry in row {} of {} is solid, but the units are {}" \ + " which is not a valid unit for this phase of matter. Solids must be measured in " \ + "units of mass." \ + .format(index, filename, units) + except: + error_message = "The phase_of_matter entry in row {} of {} is solid, but the units are {}" \ + " which is not a valid unit for this phase of matter. Solids must be measured in " \ + "units of mass." \ + .format(index, filename, units) + elif input_val == 'liquid': + # check if units are valid units for liquid (dimension of units must be volume, aka length^3) + try: + if not str(ureg(units).dimensionality) == '[length] ** 3': + error_message = "The phase_of_matter entry in row {} of {} is liquid, but the units are {}" \ + " which is not a valid unit for this phase of matter. Liquids must be measured" \ + " in units of volume." \ + .format(index, filename, units) + except: + error_message = "The phase_of_matter entry in row {} of {} is liquid, but the units are {}" \ + " which is not a valid unit for this phase of matter. Liquids must be measured" \ + " in units of volume." \ + .format(index, filename, units) + else: + # throw error that phase is neither solid nor liquid + error_message = "There is an error in the phase_of_matter entry in row {} of {}. " \ + "The entry is not one of 'solid' or 'liquid'." \ + .format(index, filename) + + elif input_type == 'commodity_quantity': + try: + float(input_val) + except ValueError: + error_message = "There is an error in the value entry in row {} of {}. " \ + "The entry is empty or non-numeric (check for extraneous characters)." \ + .format(index, filename) + + return error_message + + +# ============================================================================== + + +def load_facility_commodities_input_data(the_scenario, commodity_input_file, logger): + logger.debug("start: load_facility_commodities_input_data") + if not os.path.exists(commodity_input_file): + logger.warning("warning: cannot find commodity_input file: {}".format(commodity_input_file)) + return + + # create a temp dict to store values from CSV + temp_facility_commodities_dict = {} + + # create empty dictionary to manage schedule input + facility_schedule_dict = {} + + # read through facility_commodities input CSV + import csv + with open(commodity_input_file, 'rt') as f: + + reader = csv.DictReader(f) + # adding row index for issue #220 to alert user on which row their error is in + for index, row in enumerate(reader): + # re: issue #149 -- if the line is empty, just skip it + if list(row.values())[0] == '': + logger.debug('the CSV file has a blank in the first column. Skipping this line: {}'.format( + list(row.values()))) + continue + # {'units': 'kgal', 'facility_name': 'd:01053', 'phase_of_matter': 'liquid', 'value': '9181.521484', + # 'commodity': 'diesel', 'io': 'o', 'share_max_transport_distance'; 'Y'} + io = row["io"] + facility_name = str(row["facility_name"]) + facility_type = row["facility_type"] + commodity_name = row["commodity"].lower() # re: issue #131 - make all commodities lower case + commodity_quantity = row["value"] + commodity_unit = str(row["units"]).replace(' ', '_').lower() # remove spaces and make units lower case + commodity_phase = row["phase_of_matter"] + + # check for proc_cand-specific "non-commodities" to ignore validation (issue #254) + non_commodities = ['minsize', 'maxsize', 'cost_formula', 'min_aggregation'] + + # input data validation + if commodity_name not in non_commodities: # re: issue #254 only test actual commodities + # test io + io = io.lower() # convert 'I' and 'O' to 'i' and 'o' + error_message = check_for_input_error("io", io, commodity_input_file, index) + if error_message: + raise Exception(error_message) + # test facility type + error_message = check_for_input_error("facility_type", facility_type, commodity_input_file, index) + if error_message: + raise Exception(error_message) + # test commodity quantity + error_message = check_for_input_error("commodity_quantity", commodity_quantity, commodity_input_file, index) + if error_message: + raise Exception(error_message) + # test commodity phase + error_message = check_for_input_error("commodity_phase", commodity_phase, commodity_input_file, index, + units=commodity_unit) + if error_message: + raise Exception(error_message) + else: + logger.debug("Skipping input validation on special candidate processor commodity: {}" + .format(commodity_name)) + + if "max_processor_input" in list(row.keys()): + max_processor_input = row["max_processor_input"] + else: + max_processor_input = "Null" + + if "max_transport_distance" in list(row.keys()): + commodity_max_transport_distance = row["max_transport_distance"] + else: + commodity_max_transport_distance = "Null" + + if "share_max_transport_distance" in list(row.keys()): + share_max_transport_distance = row["share_max_transport_distance"] + else: + share_max_transport_distance = 'N' + + # add schedule_id, if available + if "schedule" in list(row.keys()): + schedule_name = str(row["schedule"]).lower() + + # blank schedule name should be cast to default + if schedule_name == "none": + schedule_name = "default" + else: + schedule_name = "default" + + # manage facility_schedule_dict + if facility_name not in facility_schedule_dict: + facility_schedule_dict[facility_name] = schedule_name + elif facility_schedule_dict[facility_name] != schedule_name: + logger.info("Schedule name '{}' does not match previously entered schedule '{}' for facility '{}'". + format(schedule_name, facility_schedule_dict[facility_name], facility_name)) + schedule_name = facility_schedule_dict[facility_name] + + # use pint to set the quantity and units + commodity_quantity_and_units = Q_(float(commodity_quantity), commodity_unit) + if max_processor_input != 'Null': + max_input_quantity_and_units = Q_(float(max_processor_input), commodity_unit) + + if commodity_phase.lower() == 'liquid': + commodity_unit = the_scenario.default_units_liquid_phase + if commodity_phase.lower() == 'solid': + commodity_unit = the_scenario.default_units_solid_phase + + if commodity_name == 'cost_formula': + pass + else: + commodity_quantity = commodity_quantity_and_units.to(commodity_unit).magnitude + + if max_processor_input != 'Null': + max_processor_input = max_input_quantity_and_units.to(commodity_unit).magnitude + + # add to the dictionary of facility_commodities mapping + if facility_name not in list(temp_facility_commodities_dict.keys()): + temp_facility_commodities_dict[facility_name] = [] + + temp_facility_commodities_dict[facility_name].append([facility_type, commodity_name, commodity_quantity, + commodity_unit, commodity_phase, + commodity_max_transport_distance, io, + share_max_transport_distance, max_processor_input, + schedule_name]) + + logger.debug("finished: load_facility_commodities_input_data") + return temp_facility_commodities_dict + + +# ============================================================================== + + +def populate_facility_commodities_table(the_scenario, commodity_input_file, logger): + + logger.debug("start: populate_facility_commodities_table {}".format(commodity_input_file)) + + if not os.path.exists(commodity_input_file): + logger.debug("note: cannot find commodity_input file: {}".format(commodity_input_file)) + return + + facility_commodities_dict = load_facility_commodities_input_data(the_scenario, commodity_input_file, logger) + + candidate = 0 + if os.path.split(commodity_input_file)[1].find("ftot_generated_processor_candidates") > -1: + candidate = 1 + + # connect to main.db and add values to table + # --------------------------------------------------------- + with sqlite3.connect(the_scenario.main_db) as db_con: + for facility_name, facility_data in iteritems(facility_commodities_dict): + + # unpack the facility_type (should be the same for all entries) + facility_type = facility_data[0][0] + facility_type_id = get_facility_id_type(the_scenario, db_con, facility_type, logger) + + location_id = get_facility_location_id(the_scenario, db_con, facility_name, logger) + + # get schedule id from the db + schedule_name = facility_data[0][-1] + schedule_id = get_schedule_id(the_scenario, db_con, schedule_name, logger) + + max_processor_input = facility_data[0][-2] + + # get the facility_id from the db (add the facility if it doesn't exists) + # and set up entry in facility_id table + facility_id = get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger) + + # iterate through each commodity + for commodity_data in facility_data: + + # get commodity_id. (adds commodity if it doesn't exist) + commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger) + + [facility_type, commodity_name, commodity_quantity, commodity_units, commodity_phase, commodity_max_transport_distance, io, share_max_transport_distance, unused_var_max_processor_input, schedule_id] = commodity_data + + if not commodity_quantity == "0.0": # skip anything with no material + sql = "insert into facility_commodities " \ + "(facility_id, location_id, commodity_id, quantity, units, io, share_max_transport_distance) " \ + "values ('{}','{}', '{}', '{}', '{}', '{}', '{}');".format( + facility_id, location_id, commodity_id, commodity_quantity, commodity_units, io, share_max_transport_distance) + db_con.execute(sql) + else: + logger.debug("skipping commodity_data {} because quantity: {}".format(commodity_name, commodity_quantity)) + db_con.execute("""update commodities + set share_max_transport_distance = + (select 'Y' from facility_commodities fc + where commodities.commodity_id = fc.commodity_id + and fc.share_max_transport_distance = 'Y') + where exists (select 'Y' from facility_commodities fc + where commodities.commodity_id = fc.commodity_id + and fc.share_max_transport_distance = 'Y') + ;""" + ) + + logger.debug("finished: populate_facility_commodities_table") + + +# ============================================================================== + + +def db_check_multiple_input_commodities_for_processor(the_scenario, logger): + # connect to main.db and add values to table + # --------------------------------------------------------- + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select f.facility_name, count(*) + from facility_commodities fc + join facilities f on f.facility_ID = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where fti.facility_type like 'processor' and fc.io like 'i' + group by f.facility_name + having count(*) > 1;""" + db_cur = db_con.execute(sql) + data = db_cur.fetchall() + if len(data) > 0: + for multi_input_processor in data: + logger.warning("Processor: {} has {} input commodities specified.".format(multi_input_processor[0], + multi_input_processor[1])) + logger.warning("Multiple processor inputs are not supported in the same scenario as shared max transport " + "distance") + # logger.info("make adjustments to the processor commodity input file: {}".format(the_scenario.processors_commodity_data)) + # error = "Multiple input commodities for processors is not supported in FTOT" + # logger.error(error) + # raise Exception(error) + + +# ============================================================================== + + +def populate_coprocessing_table(the_scenario, logger): + + logger.info("start: populate_coprocessing_table") + + # connect to db + with sqlite3.connect(the_scenario.main_db) as db_con: + + # should be filled from the file coprocessing.csv, which needs to be added to xml still + # I would place this file in the common data folder probably, since it is a fixed reference table + # for now, filling the db table here manually with the data from the csv file + sql = """INSERT INTO coprocessing (coproc_id, label, description) + VALUES + (1, 'single', 'code should throw error if processor has more than one input commodity'), + (2, 'fixed combination', 'every input commodity listed for the processor is required, in the ratio ' || + 'specified by their quantities. Output requires all inputs to be present; '), + (3, 'substitutes allowed', 'any one of the input commodities listed for the processor can be used ' || + 'to generate the output, with the ratios specified by quantities. A ' || + 'combination of inputs is allowed. '), + (4, 'external input', 'assumes all specified inputs are required, in that ratio, with the addition ' || + 'of an External or Non-Transported input commodity. This is included in the ' || + 'ratio and as part of the input capacity, but is available in unlimited ' || + 'quantity at the processor location. ') + ; """ + db_con.execute(sql) + + logger.debug("not yet implemented: populate_coprocessing_table") + + +# ============================================================================= + + +def populate_locations_table(the_scenario, logger): + + logger.info("start: populate_locations_table") + + # connect to db + with sqlite3.connect(the_scenario.main_db) as db_con: + + # then iterate through the GIS and get the facility_name, shape_x and shape_y + with arcpy.da.Editor(the_scenario.main_gdb) as edit: + + logger.debug("iterating through the FC and create a facility_location mapping with x,y coord.") + + for fc in [the_scenario.rmp_fc, the_scenario.destinations_fc, the_scenario.processors_fc]: + + logger.debug("iterating through the FC: {}".format(fc)) + with arcpy.da.SearchCursor(fc, ["facility_name", "SHAPE@X", "SHAPE@Y"]) as cursor: + for row in cursor: + facility_name = row[0] + shape_x = round(row[1], -2) + shape_y = round(row[2], -2) + + # check if location_id exists exists for snap_x and snap_y + location_id = get_location_id(the_scenario, db_con, shape_x, shape_y, logger) + + if location_id > 0: + # map facility_name to the location_id in the tmp_facility_locations table + db_con.execute("insert or ignore into tmp_facility_locations " + "(facility_name, location_id) " + "values ('{}', '{}');".format(facility_name, location_id)) + else: + error = "no location_id exists for shape_x {} and shape_y {}".format(shape_x, shape_y) + logger.error(error) + raise Exception(error) + + logger.debug("finished: populate_locations_table") + + +# ============================================================================= + + +def get_location_id(the_scenario, db_con, shape_x, shape_y, logger): + + location_id = None + + # get location_id + for row in db_con.execute(""" + select + location_id + from locations l + where l.shape_x = '{}' and l.shape_y = '{}';""".format(shape_x, shape_y)): + + location_id = row[0] + + # if no ID, add it. + if location_id is None: + # if it doesn't exist, add to locations table and generate a location id. + db_cur = db_con.execute("insert into locations (shape_x, shape_y) values ('{}', '{}');".format(shape_x, shape_y)) + location_id = db_cur.lastrowid + + # check again. we should have the ID now. if we don't throw an error. + if location_id is None: + error = "something went wrong getting location_id shape_x: {}, shape_y: {} location_id ".format(shape_x, shape_y) + logger.error(error) + raise Exception(error) + else: + return location_id + + +# ============================================================================= + + +def get_facility_location_id(the_scenario, db_con, facility_name, logger): + + # get location_id + db_cur = db_con.execute("select location_id from tmp_facility_locations l where l.facility_name = '{}';".format(str(facility_name))) + location_id = db_cur.fetchone() + if not location_id: + warning = "location_id for tmp_facility_name: {} is not found.".format(facility_name) + logger.debug(warning) + else: + return location_id[0] + + +# ============================================================================= + + +def get_facility_id(the_scenario, db_con, location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input, logger): + + # if it doesn't exist, add to facilities table and generate a facility id. + db_con.execute("insert or ignore into facilities " + "(location_id, facility_name, facility_type_id, candidate, schedule_id, max_capacity) " + "values ('{}', '{}', {}, {}, {}, {});".format(location_id, facility_name, facility_type_id, candidate, schedule_id, max_processor_input)) + + # get facility_id + db_cur = db_con.execute("select facility_id " + "from facilities f " + "where f.facility_name = '{}' and facility_type_id = {};".format(facility_name, facility_type_id)) + facility_id = db_cur.fetchone()[0] + + if not facility_id: + error = "something went wrong getting {} facility_id ".format(facility_id) + logger.error(error) + raise Exception(error) + else: + return facility_id + + +# =================================================================================================== + + +def get_facility_id_type(the_scenario, db_con, facility_type, logger): + facility_type_id = None + + # get facility_id_type + for row in db_con.execute("select facility_type_id " + "from facility_type_id f " + "where facility_type = '{}';".format(facility_type)): + facility_type_id = row[0] + + # if it doesn't exist, add to facility_type table and generate a facility_type_id. + if facility_type_id is None: + db_cur = db_con.execute("insert into facility_type_id (facility_type) values ('{}');".format(facility_type)) + facility_type_id = db_cur.lastrowid + + # check again if we have facility_type_id + if facility_type_id is None: + error = "something went wrong getting {} facility_type_id ".format(facility_type) + logger.error(error) + raise Exception(error) + else: + return facility_type_id + + +# =================================================================================================== + + +def get_commodity_id(the_scenario, db_con, commodity_data, logger): + + [facility_type, commodity_name, commodity_quantity, commodity_unit, commodity_phase, + commodity_max_transport_distance, io, share_max_transport_distance, max_processor_input, schedule_id] = commodity_data + + # get the commodity_id. + db_cur = db_con.execute("select commodity_id " + "from commodities c " + "where c.commodity_name = '{}';".format(commodity_name)) + + commodity_id = db_cur.fetchone() # don't index the list, since it might not exists. + + if not commodity_id: + # if it doesn't exist, add the commodity to the commodities table and generate a commodity id + if commodity_max_transport_distance in ['Null', '', 'None']: + sql = "insert into commodities " \ + "(commodity_name, units, phase_of_matter, share_max_transport_distance) " \ + "values ('{}', '{}', '{}','{}');".format(commodity_name, commodity_unit, commodity_phase, + share_max_transport_distance) + else: + sql = "insert into commodities " \ + "(commodity_name, units, phase_of_matter, max_transport_distance, share_max_transport_distance) " \ + "values ('{}', '{}', '{}', {}, '{}');".format(commodity_name, commodity_unit, commodity_phase, + commodity_max_transport_distance, + share_max_transport_distance) + db_con.execute(sql) + + # get the commodity_id. + db_cur = db_con.execute("select commodity_id " + "from commodities c " + "where c.commodity_name = '{}';".format(commodity_name)) + commodity_id = db_cur.fetchone()[0] # index the first (and only) data in the list + + if not commodity_id: + error = "something went wrong adding the commodity {} " \ + "to the commodities table and getting a commodity_id".format(commodity_name) + logger.error(error) + raise Exception(error) + else: + return commodity_id + + +# =================================================================================================== + + +def get_schedule_id(the_scenario, db_con, schedule_name, logger): + # get location_id + db_cur = db_con.execute( + "select schedule_id from schedule_names s where s.schedule_name = '{}';".format(str(schedule_name))) + schedule_id = db_cur.fetchone() + if not schedule_id: + # if schedule id is not found, replace with the default schedule + warning = 'schedule_id for schedule_name: {} is not found. Replace with default'.format(schedule_name) + logger.info(warning) + db_cur = db_con.execute( + "select schedule_id from schedule_names s where s.schedule_name = 'default';") + schedule_id = db_cur.fetchone() + + return schedule_id[0] + + +# =================================================================================================== + + +def gis_clean_fc(the_scenario, logger): + + logger.info("start: gis_clean_fc") + + start_time = datetime.datetime.now() + + # clear the destinations + gis_clear_feature_class(the_scenario.destinations_fc, logger) + + # clear the RMPs + gis_clear_feature_class(the_scenario.rmp_fc, logger) + + # clear the processors + gis_clear_feature_class(the_scenario.processors_fc, logger) + + # clear the processors + gis_clear_feature_class(the_scenario.locations_fc, logger) + + logger.debug("finished: gis_clean_fc: Runtime (HMS): \t{}".format + (ftot_supporting.get_total_runtime_string(start_time))) + + +# ============================================================================== + + +def gis_clear_feature_class(fc, logger): + logger.debug("start: clear_feature_class for fc {}".format(os.path.split(fc)[1])) + if arcpy.Exists(fc): + arcpy.Delete_management(fc) + logger.debug("finished: deleted existing fc {}".format(os.path.split(fc)[1])) + + +# =================================================================================================== + + +def gis_get_feature_count(fc, logger): + result = arcpy.GetCount_management(fc) + count = int(result.getOutput(0)) + return count + + +# =================================================================================================== + + +def gis_populate_fc(the_scenario, logger): + + logger.info("start: gis_populate_fc") + + start_time = datetime.datetime.now() + + # populate the destinations fc in main.gdb + gis_ultimate_destinations_setup_fc(the_scenario, logger) + + # populate the RMPs fc in main.gdb + gis_rmp_setup_fc(the_scenario, logger) + + # populate the processors fc in main.gdb + gis_processors_setup_fc(the_scenario, logger) + + logger.debug("finished: gis_populate_fc: Runtime (HMS): \t{}".format + (ftot_supporting.get_total_runtime_string(start_time))) + + +# ------------------------------------------------------------ + + +def gis_ultimate_destinations_setup_fc(the_scenario, logger): + + logger.info("start: gis_ultimate_destinations_setup_fc") + + start_time = datetime.datetime.now() + + # copy the destination from the baseline layer to the scenario gdb + # -------------------------------------------------------------- + if not arcpy.Exists(the_scenario.base_destination_layer): + error = "can't find baseline data destinations layer {}".format(the_scenario.base_destination_layer) + raise IOError(error) + + destinations_fc = the_scenario.destinations_fc + arcpy.Project_management(the_scenario.base_destination_layer, destinations_fc, ftot_supporting_gis.LCC_PROJ) + + # Delete features with no data in csv -- cleans up GIS output and eliminates unnecessary GIS processing + # -------------------------------------------------------------- + # create a temp dict to store values from CSV + temp_facility_commodities_dict = {} + counter = 0 + + # read through facility_commodities input CSV + import csv + with open(the_scenario.destinations_commodity_data, 'rt') as f: + reader = csv.DictReader(f) + for row in reader: + facility_name = str(row["facility_name"]) + commodity_quantity = float(row["value"]) + + if facility_name not in list(temp_facility_commodities_dict.keys()): + if commodity_quantity > 0: + temp_facility_commodities_dict[facility_name] = True + + with arcpy.da.UpdateCursor(destinations_fc, ['Facility_Name']) as cursor: + for row in cursor: + if row[0] in temp_facility_commodities_dict: + pass + else: + cursor.deleteRow() + counter += 1 + del cursor + logger.config("Number of Destinations removed due to lack of commodity data: \t{}".format(counter)) + + with arcpy.da.SearchCursor(destinations_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: + for row in scursor: + # Check if coordinates of facility are roughly within North America + if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: + pass + else: + logger.warning("Facility: {} is not located in North America.".format(row[0])) + logger.info("remove the facility from the scenario or make adjustments to the facility's location in" + " the destinations feature class: {}".format(the_scenario.base_destination_layer)) + error = "Facilities outside North America are not supported in FTOT" + logger.error(error) + raise Exception(error) + + result = gis_get_feature_count(destinations_fc, logger) + logger.config("Number of Destinations: \t{}".format(result)) + + logger.debug("finish: gis_ultimate_destinations_setup_fc: Runtime (HMS): \t{}".format + (ftot_supporting.get_total_runtime_string(start_time))) + + +# ============================================================================= + + +def gis_rmp_setup_fc(the_scenario, logger): + + logger.info("start: gis_rmp_setup_fc") + start_time = datetime.datetime.now() + + # copy the rmp from the baseline data to the working gdb + # ---------------------------------------------------------------- + if not arcpy.Exists(the_scenario.base_rmp_layer): + error = "can't find baseline data rmp layer {}".format(the_scenario.base_rmp_layer) + raise IOError(error) + + rmp_fc = the_scenario.rmp_fc + arcpy.Project_management(the_scenario.base_rmp_layer, rmp_fc, ftot_supporting_gis.LCC_PROJ) + + # Delete features with no data in csv-- cleans up GIS output and eliminates unnecessary GIS processing + # -------------------------------------------------------------- + # create a temp dict to store values from CSV + temp_facility_commodities_dict = {} + counter = 0 + + # read through facility_commodities input CSV + import csv + with open(the_scenario.rmp_commodity_data, 'rt') as f: + + reader = csv.DictReader(f) + for row in reader: + facility_name = str(row["facility_name"]) + commodity_quantity = float(row["value"]) + + if facility_name not in list(temp_facility_commodities_dict.keys()): + if commodity_quantity > 0: + temp_facility_commodities_dict[facility_name] = True + + with arcpy.da.UpdateCursor(rmp_fc, ['Facility_Name']) as cursor: + for row in cursor: + if row[0] in temp_facility_commodities_dict: + pass + else: + cursor.deleteRow() + counter +=1 + del cursor + logger.config("Number of RMPs removed due to lack of commodity data: \t{}".format(counter)) + + with arcpy.da.SearchCursor(rmp_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: + for row in scursor: + # Check if coordinates of facility are roughly within North America + if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: + pass + else: + logger.warning("Facility: {} is not located in North America.".format(row[0])) + logger.info("remove the facility from the scenario or make adjustments to the facility's location in " + "the RMP feature class: {}".format(the_scenario.base_rmp_layer)) + error = "Facilities outside North America are not supported in FTOT" + logger.error(error) + raise Exception(error) + + del scursor + + result = gis_get_feature_count(rmp_fc, logger) + logger.config("Number of RMPs: \t{}".format(result)) + + logger.debug("finished: gis_rmp_setup_fc: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + +# ============================================================================= + + +def gis_processors_setup_fc(the_scenario, logger): + + logger.info("start: gis_processors_setup_fc") + start_time = datetime.datetime.now() + + if str(the_scenario.base_processors_layer).lower() == "null" or \ + str(the_scenario.base_processors_layer).lower() == "none": + # create an empty processors layer + # ------------------------- + processors_fc = the_scenario.processors_fc + + if arcpy.Exists(processors_fc): + arcpy.Delete_management(processors_fc) + logger.debug("deleted existing {} layer".format(processors_fc)) + + arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "processors", "POINT", "#", "DISABLED", "DISABLED", + ftot_supporting_gis.LCC_PROJ, "#", "0", "0", "0") + + arcpy.AddField_management(processors_fc, "Facility_Name", "TEXT", "#", "#", "25", "#", "NULLABLE", + "NON_REQUIRED", "#") + arcpy.AddField_management(processors_fc, "Candidate", "SHORT") + + else: + # copy the processors from the baseline data to the working gdb + # ---------------------------------------------------------------- + if not arcpy.Exists(the_scenario.base_processors_layer): + error = "can't find baseline data processors layer {}".format(the_scenario.base_processors_layer) + raise IOError(error) + + processors_fc = the_scenario.processors_fc + arcpy.Project_management(the_scenario.base_processors_layer, processors_fc, ftot_supporting_gis.LCC_PROJ) + + arcpy.AddField_management(processors_fc, "Candidate", "SHORT") + + # Delete features with no data in csv-- cleans up GIS output and eliminates unnecessary GIS processing + # -------------------------------------------------------------- + # create a temp dict to store values from CSV + temp_facility_commodities_dict = {} + counter = 0 + + # read through facility_commodities input CSV + import csv + with open(the_scenario.processors_commodity_data, 'rt') as f: + + reader = csv.DictReader(f) + for row in reader: + facility_name = str(row["facility_name"]) + commodity_quantity = float(row["value"]) + + if facility_name not in list(temp_facility_commodities_dict.keys()): + if commodity_quantity > 0: + temp_facility_commodities_dict[facility_name] = True + + with arcpy.da.UpdateCursor(processors_fc, ['Facility_Name']) as cursor: + for row in cursor: + if row[0] in temp_facility_commodities_dict: + pass + else: + cursor.deleteRow() + counter += 1 + + del cursor + logger.config("Number of processors removed due to lack of commodity data: \t{}".format(counter)) + + with arcpy.da.SearchCursor(processors_fc, ['Facility_Name', 'SHAPE@X', 'SHAPE@Y']) as scursor: + for row in scursor: + # Check if coordinates of facility are roughly within North America + if -6500000 < row[1] < 6500000 and -3000000 < row[2] < 5000000: + pass + else: + logger.warning("Facility: {} is not located in North America.".format(row[0])) + logger.info("remove the facility from the scenario or make adjustments to the facility's location " + "in the processors feature class: {}".format(the_scenario.base_processors_layer)) + error = "Facilities outside North America are not supported in FTOT" + logger.error(error) + raise Exception(error) + + del scursor + + # check for candidates or other processors specified in either XML or + layers_to_merge = [] + + # add the candidates_for_merging if they exists. + if arcpy.Exists(the_scenario.processor_candidates_fc): + logger.info("adding {} candidate processors to the processors fc".format( + gis_get_feature_count(the_scenario.processor_candidates_fc, logger))) + layers_to_merge.append(the_scenario.processor_candidates_fc) + gis_merge_processor_fc(the_scenario, layers_to_merge, logger) + + result = gis_get_feature_count(processors_fc, logger) + + logger.config("Number of Processors: \t{}".format(result)) + + logger.debug("finish: gis_processors_setup_fc: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + +# ======================================================================= + + +def gis_merge_processor_fc(the_scenario, layers_to_merge, logger): + + logger.info("merge the candidates and processors together. ") + + scenario_gdb = the_scenario.main_gdb + + # ------------------------------------------------------------------------- + # append the processors from the kayers to merge data to the working gdb + # ------------------------------------------------------------------------- + if len(layers_to_merge) == 0: + error = "len of layers_to_merge for processors is {}".format(layers_to_merge) + raise IOError(error) + + processors_fc = the_scenario.processors_fc + + # Open an edit session and start an edit operation + with arcpy.da.Editor(scenario_gdb) as edit: + # Report out some information about how many processors are in each layer to merge + for layer in layers_to_merge: + + processor_count = str(arcpy.GetCount_management(layer)) + logger.debug("Importing {} processors from {}".format(processor_count, os.path.split(layer)[0])) + logger.info("Append candidates into the Processors FC") + + arcpy.Append_management(layer, processors_fc, "NO_TEST") + + total_processor_count = str(arcpy.GetCount_management(processors_fc)) + logger.debug("Total count: {} records in {}".format(total_processor_count, os.path.split(processors_fc)[0])) + + return + diff --git a/program/ftot_maps.py b/program/ftot_maps.py index d2d506c..95637a6 100644 --- a/program/ftot_maps.py +++ b/program/ftot_maps.py @@ -1,1029 +1,1029 @@ - -# --------------------------------------------------------------------------------------------------- -# Name: ftot_maps -# --------------------------------------------------------------------------------------------------- -import os -import arcpy -from shutil import copy -import imageio -import sqlite3 -import datetime -from six import iteritems - - -# =================================================================================================== -def new_map_creation(the_scenario, logger, task): - - logger.info("start: maps") - - # create map directory - timestamp_folder_name = 'maps_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") - - if task == "m": - basemap = "default_basemap" - elif task == "mb": - basemap = "gray_basemap" - elif task == "mc": - basemap = "topo_basemap" - elif task == "md": - basemap = "street_basemap" - else: - basemap = "default_basemap" - - the_scenario.mapping_directory = os.path.join(the_scenario.scenario_run_directory, "Maps", - basemap + "_" + timestamp_folder_name) - scenario_gdb = the_scenario.main_gdb - - scenario_aprx_location = os.path.join(the_scenario.scenario_run_directory, "Maps", "ftot_maps.aprx") - if not os.path.exists(the_scenario.mapping_directory): - logger.debug("creating maps directory.") - os.makedirs(the_scenario.mapping_directory) - - # Must delete existing scenario aprx because there can only be one per scenario (everything in the aprx references - # the existing scenario main.gdb) - if os.path.exists(scenario_aprx_location): - os.remove(scenario_aprx_location) - - # set aprx locations: root is in the common data folder, and the scenario is in the local maps folder - root_ftot_aprx_location = os.path.join(the_scenario.common_data_folder, "ftot_maps.aprx") - base_layers_location = os.path.join(the_scenario.common_data_folder, "Base_Layers") - - # copy the aprx from the common data director to the scenario maps directory - logger.debug("copying the root aprx from common data to the scenario maps directory.") - copy(root_ftot_aprx_location, scenario_aprx_location) - - # load the map document into memory - aprx = arcpy.mp.ArcGISProject(scenario_aprx_location) - - # Fix the non-base layers here - broken_list = aprx.listBrokenDataSources() - for broken_item in broken_list: - if broken_item.supports("DATASOURCE"): - if not broken_item.longName.find("Base") == 0: - conprop = broken_item.connectionProperties - conprop['connection_info']['database'] = scenario_gdb - broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) - - list_broken_data_sources(aprx, base_layers_location, logger) - - reset_map_base_layers(aprx, logger, basemap) - - export_map_steps(aprx, the_scenario, logger, basemap) - - logger.info("maps located here: {}".format(the_scenario.mapping_directory)) - - -# =================================================================================================== -def list_broken_data_sources(aprx, base_layers_location, logger): - broken_list = aprx.listBrokenDataSources() - for broken_item in broken_list: - if broken_item.supports("DATASOURCE"): - if broken_item.longName.find("Base") == 0: - conprop = broken_item.connectionProperties - conprop['connection_info']['database'] = base_layers_location - broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) - logger.debug("\t" + "broken base layer path fixed: " + broken_item.name) - else: - logger.debug("\t" + "broken aprx data source path: " + broken_item.name) - - -# =================================================================================================== -def reset_map_base_layers(aprx, logger, basemap): - - logger.debug("start: reset_map_base_layers") - # turn on base layers, turn off everything else. - map = aprx.listMaps("ftot_map")[0] - for lyr in map.listLayers(): - - if lyr.longName.find("Base") == 0: - lyr.visible = True - - # This group layer should be off unless mb step is being run - if basemap not in ["gray_basemap"] and "World Light Gray" in lyr.longName and \ - not lyr.longName.endswith("Light Gray Canvas Base"): - lyr.visible = False - - # This group layer should be off unless mc step is being run - if basemap not in ["topo_basemap"] and "USGSTopo" in lyr.longName and not lyr.longName.endswith("Layers"): - lyr.visible = False - - # This group layer should be off unless md step is being run - if basemap not in ["street_basemap"] and "World Street Map" in lyr.longName and not lyr.longName.endswith("Layers"): - lyr.visible = False - - # These layers can't be turned off. So leave them on. - elif lyr.longName.endswith("Light Gray Canvas Base"): - lyr.visible = True - elif lyr.longName == "Layers": - lyr.visible = True - - else: - lyr.visible = False - - layout = aprx.listLayouts("ftot_layout")[0] - elm = layout.listElements("TEXT_ELEMENT")[0] - elm.text = " " - - return - - -# =================================================================================================== -def get_layer_dictionary(aprx, logger): - logger.debug("start: get_layer_dictionary") - layer_dictionary = {} - map = aprx.listMaps("ftot_map")[0] - for lyr in map.listLayers(): - - layer_dictionary[lyr.longName.replace(" ", "_").replace("\\", "_")] = lyr - - return layer_dictionary - - -# =================================================================================================== -def debug_layer_status(aprx, logger): - - layer_dictionary = get_layer_dictionary(aprx, logger) - - for layer_name, lyr in sorted(iteritems(layer_dictionary)): - - logger.info("layer: {}, visible: {}".format(layer_name, lyr.visible)) - - -# ===================================================================================================- -def export_to_png(map_name, aprx, the_scenario, logger): - - file_name = str(map_name + ".png").replace(" ", "_").replace("\\", "_") - - file_location = os.path.join(the_scenario.mapping_directory, file_name) - - layout = aprx.listLayouts("ftot_layout")[0] - - # # Setup legend properties so everything refreshes as expected - legend = layout.listElements("LEGEND_ELEMENT", "Legend")[0] - legend.fittingStrategy = 'AdjustFrame' - legend.syncLayerVisibility = True - legend.syncLayerOrder = True - aprx.save() - - layout.exportToPNG(file_location) - - logger.info("exporting: {}".format(map_name)) - - -# =================================================================================================== -def export_map_steps(aprx, the_scenario, logger, basemap): - - # ------------------------------------------------------------------------------------ - - # Project and zoom to extent of features - # Get extent of the locations and optimized_route_segments FCs and zoom to buffered extent - - # A list of extents - extent_list = [] - - for fc in [os.path.join(the_scenario.main_gdb, "locations"), - os.path.join(the_scenario.main_gdb, "optimized_route_segments")]: - # Cycle through layers grabbing extents, converting them into - # polygons and adding them to extentList - desc = arcpy.Describe(fc) - ext = desc.extent - array = arcpy.Array([ext.upperLeft, ext.upperRight, ext.lowerRight, ext.lowerLeft]) - extent_list.append(arcpy.Polygon(array)) - sr = desc.spatialReference - - # Create a temporary FeatureClass from the polygons - arcpy.CopyFeatures_management(extent_list, r"in_memory\temp") - - # Get extent of this temporary layer and zoom to its extent - desc = arcpy.Describe(r"in_memory\temp") - extent = desc.extent - - ll_geom = arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(arcpy.SpatialReference(4326)) - ur_geom = arcpy.PointGeometry(extent.upperRight, sr).projectAs(arcpy.SpatialReference(4326)) - - ll = ll_geom.centroid - ur = ur_geom.centroid - - new_sr = create_custom_spatial_ref(ll, ur) - - set_extent(aprx, extent, sr, new_sr) - - # Clean up - arcpy.Delete_management(r"in_memory\temp") - del ext, desc, array, extent_list - - # Save aprx so that after step is run user can open the aprx at the right zoom/ extent to continue examining the data. - aprx.save() - - # reset the map so we are working from a clean and known starting point. - reset_map_base_layers(aprx, logger, basemap) - - # get a dictionary of all the layers in the aprx - # might want to get a list of groups - layer_dictionary = get_layer_dictionary(aprx, logger) - - logger.debug("layer_dictionary.keys(): \t {}".format(list(layer_dictionary.keys()))) - - # create a variable for each layer so we can access each layer easily - - s_step_lyr = layer_dictionary["S_STEP"] - f_step_rmp_lyr = layer_dictionary["F_STEP_RMP"] - f_step_proc_lyr = layer_dictionary["F_STEP_PROC"] - f_step_dest_lyr = layer_dictionary["F_STEP_DEST"] - f_step_lyr = layer_dictionary["F_STEP"] - f_step_rmp_labels_lyr = layer_dictionary["F_STEP_RMP_W_LABELS"] - f_step_proc_labels_lyr = layer_dictionary["F_STEP_PROC_W_LABELS"] - f_step_dest_labels_lyr = layer_dictionary["F_STEP_DEST_W_LABELS"] - f_step_labels_lyr = layer_dictionary["F_STEP_W_LABELS"] - o_step_opt_flow_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW"] - o_step_opt_flow_no_labels_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW_NO_LABELS"] - o_step_opt_flow_just_flow_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW_JUST_FLOW"] - o_step_rmp_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_RMP_OPTIMAL_VS_NON_OPTIMAL"] - o_step_proc_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_PROC_OPTIMAL_VS_NON_OPTIMAL"] - o_step_dest_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_DEST_OPTIMAL_VS_NON_OPTIMAL"] - o_step_raw_producer_to_processor_lyr = layer_dictionary["O_STEP_RMP_TO_PROC"] - o_step_processor_to_destination_lyr = layer_dictionary["O_STEP_PROC_TO_DEST"] - f2_step_candidates_lyr = layer_dictionary["F2_STEP_CANDIDATES"] - f2_step_merged_lyr = layer_dictionary["F2_STEP_MERGE"] - f2_step_candidates_labels_lyr = layer_dictionary["F2_STEP_CANDIDATES_W_LABELS"] - f2_step_merged_labels_lyr = layer_dictionary["F2_STEP_MERGE_W_LABELS"] - - # Custom user-created maps-- user can create group layers within this parent group layer containing different - # features they would like to map. Code will look for this group layer name, which should not change. - if "CUSTOM_USER_CREATED_MAPS" in layer_dictionary: - custom_maps_parent_lyr = layer_dictionary["CUSTOM_USER_CREATED_MAPS"] - else: - custom_maps_parent_lyr = None - - # START MAKING THE MAPS! - -# turn off all the groups -# turn on the group step -# set caption information -# Optimal Processor to Optimal Ultimate Destination Delivery Routes -# opt_destinations_lyr.visible = True -# map_name = "" -# caption = "" -# call generate_map() -# generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # S_STEP - s_step_lyr.visible = True - sublayers = s_step_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "01_S_Step_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_RMP - f_step_rmp_lyr.visible = True - sublayers = f_step_rmp_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02a_F_Step_RMP_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_PROC - f_step_proc_lyr.visible = True - sublayers = f_step_proc_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02b_F_Step_User_Defined_PROC_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_DEST - f_step_dest_lyr.visible = True - sublayers = f_step_dest_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02c_F_Step_DEST_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP - f_step_lyr.visible = True - sublayers = f_step_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02d_F_Step_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_RMP_LABELS - f_step_rmp_labels_lyr.visible = True - sublayers = f_step_rmp_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02a_F_Step_RMP_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_PROC_LABELS - f_step_proc_labels_lyr.visible = True - sublayers = f_step_proc_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02b_F_Step_User_Defined_PROC_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_DEST_LABELS - f_step_dest_labels_lyr.visible = True - sublayers = f_step_dest_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02c_F_Step_DEST_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F_STEP_LABELS - f_step_labels_lyr.visible = True - sublayers = f_step_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "02d_F_Step_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW - # O_STEP - A - - o_step_opt_flow_lyr.visible = True - sublayers = o_step_opt_flow_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04a_O_Step_Final_Optimal_Routes_With_Commodity_Flow_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP - B - same as above but with no labels to make it easier to see routes. - o_step_opt_flow_no_labels_lyr.visible = True - sublayers = o_step_opt_flow_no_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04b_O_Step_Final_Optimal_Routes_With_Commodity_Flow_NO_LABELS_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP - C - just flows. no Origins or Destinations - o_step_opt_flow_just_flow_lyr.visible = True - sublayers = o_step_opt_flow_just_flow_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04c_O_Step_Final_Optimal_Routes_With_Commodity_Flow_JUST_FLOW_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_OPTIMAL_AND_NON_OPTIMAL_RMP - o_step_rmp_opt_vs_non_opt_lyr.visible = True - sublayers = o_step_rmp_opt_vs_non_opt_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04d_O_Step_Optimal_and_Non_Optimal_RMP_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_OPTIMAL_AND_NON_OPTIMAL_DEST - o_step_proc_opt_vs_non_opt_lyr.visible = True - sublayers = o_step_proc_opt_vs_non_opt_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04e_O_Step_Optimal_and_Non_Optimal_PROC_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_OPTIMAL_AND_NON_OPTIMAL_DEST - o_step_dest_opt_vs_non_opt_lyr.visible = True - sublayers = o_step_dest_opt_vs_non_opt_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "04f_O_Step_Optimal_and_Non_Optimal_DEST_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # check if processor fc exists. if it doesn't stop here since there won't be any more relevant maps to make - - processor_merged_fc = the_scenario.processors_fc - processor_fc_feature_count = get_feature_count(processor_merged_fc, logger) - - if processor_fc_feature_count: - logger.debug("Processor Feature Class has {} features....".format(processor_fc_feature_count)) - # F2_STEP_CANDIDATES - f2_step_candidates_lyr.visible = True - sublayers = f2_step_candidates_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "03a_F2_Step_Processor_Candidates_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F2_STEP_MERGE - f2_step_merged_lyr.visible = True - sublayers = f2_step_merged_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "03b_F2_Step_Processors_All_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F2_STEP_CANDIDATES_W_LABELS - f2_step_candidates_labels_lyr.visible = True - sublayers = f2_step_candidates_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "03a_F2_Step_Processor_Candidates_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # F2_STEP_MERGE_W_LABELS - f2_step_merged_labels_lyr.visible = True - sublayers = f2_step_merged_labels_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "03b_F2_Step_Processors_All_With_Labels_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_RAW_MATERIAL_PRODUCER_TO_PROCESSOR - o_step_raw_producer_to_processor_lyr.visible = True - sublayers = o_step_raw_producer_to_processor_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "05a_O_Step_Raw_Material_Producer_To_Processor_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - # O_STEP_PROCESSOR_TO_ULTIMATE_DESTINATION - o_step_processor_to_destination_lyr.visible = True - sublayers = o_step_processor_to_destination_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - map_name = "05b_O_Step_Processor_To_Ultimate_Destination_" + basemap - caption = "" - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - else: - logger.info("processor layer has {} features. skipping the rest of the mapping".format - (processor_fc_feature_count)) - - # CUSTOM_MAPS - if "CUSTOM_USER_CREATED_MAPS" in layer_dictionary: - sub_groups_custom_maps = custom_maps_parent_lyr.listLayers() - for sub_group in sub_groups_custom_maps: - # Need to limit maps to just the group layers which are NOT the parent group layer. - if sub_group.isGroupLayer and sub_group != custom_maps_parent_lyr: - # First ensure that the group layer is on as this as all layers are turned off at beginning by default - custom_maps_parent_lyr.visible = True - # Then turn on the individual custom map - sub_group.visible = True - count = 0 - sublayers = sub_group.listLayers() - for sublayer in sublayers: - sublayer.visible = True - count += 1 - map_name = str(sub_group) + "_" + basemap - caption = "" - # Only generate map if there are actual features inside the group layer - if count > 0: - generate_map(caption, map_name, aprx, the_scenario, logger, basemap) - - -# =================================================================================================== -def generate_map(caption, map_name, aprx, the_scenario, logger, basemap): - import datetime - - # create a text element on the aprx for the caption - layout = aprx.listLayouts("ftot_layout")[0] - elm = layout.listElements("TEXT_ELEMENT")[0] - - # set the caption text - dt = datetime.datetime.now() - - elm.text = "Scenario Name: " + the_scenario.scenario_name + " -- Date: " + str(dt.strftime("%b %d, %Y")) + "\n" + \ - caption - - # programmatically set the caption height - if len(caption) < 180: - elm.elementHeight = 0.5 - - else: - elm.elementHeight = 0.0017*len(caption) + 0.0 - - # export that map to png - export_to_png(map_name, aprx, the_scenario, logger) - - # reset the map layers - reset_map_base_layers(aprx, logger, basemap) - - -# =================================================================================================== -def prepare_time_commodity_subsets_for_mapping(the_scenario, logger, task): - - logger.info("start: time and commodity maps") - - if task == "m2": - basemap = "default_basemap" - elif task == "m2b": - basemap = "gray_basemap" - elif task == "m2c": - basemap = "topo_basemap" - elif task == "m2d": - basemap = "street_basemap" - else: - basemap = "default_basemap" - - timestamp_folder_name = 'maps_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") - the_scenario.mapping_directory = os.path.join(the_scenario.scenario_run_directory, "Maps_Time_Commodity", - basemap + "_" + timestamp_folder_name) - scenario_aprx_location = os.path.join(the_scenario.scenario_run_directory, "Maps_Time_Commodity", "ftot_maps.aprx") - scenario_gdb = the_scenario.main_gdb - arcpy.env.workspace = scenario_gdb - - if not os.path.exists(the_scenario.mapping_directory): - logger.debug("creating maps directory.") - os.makedirs(the_scenario.mapping_directory) - - if os.path.exists(scenario_aprx_location): - os.remove(scenario_aprx_location) - - # set aprx locations: root is in the common data folder, and the scenario is in the local maps folder - root_ftot_aprx_location = os.path.join(the_scenario.common_data_folder, "ftot_maps.aprx") - base_layers_location = os.path.join(the_scenario.common_data_folder, "Base_Layers") - - # copy the aprx from the common data director to the scenario maps directory - logger.debug("copying the root aprx from common data to the scenario maps directory.") - copy(root_ftot_aprx_location, scenario_aprx_location) - - # load the map document into memory - aprx = arcpy.mp.ArcGISProject(scenario_aprx_location) - - # Fix the non-base layers here - broken_list = aprx.listBrokenDataSources() - for broken_item in broken_list: - if broken_item.supports("DATASOURCE"): - if not broken_item.longName.find("Base") == 0: - conprop = broken_item.connectionProperties - conprop['connection_info']['database'] = scenario_gdb - broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) - - list_broken_data_sources(aprx, base_layers_location, logger) - - # Project and zoom to extent of features - # Get extent of the locations and optimized_route_segments FCs and zoom to buffered extent - - # A list of extents - extent_list = [] - - for fc in [os.path.join(the_scenario.main_gdb, "locations"), - os.path.join(the_scenario.main_gdb, "optimized_route_segments")]: - # Cycle through layers grabbing extents, converting them into - # polygons and adding them to extentList - desc = arcpy.Describe(fc) - ext = desc.extent - array = arcpy.Array([ext.upperLeft, ext.upperRight, ext.lowerRight, ext.lowerLeft]) - extent_list.append(arcpy.Polygon(array)) - sr = desc.spatialReference - - # Create a temporary FeatureClass from the polygons - arcpy.CopyFeatures_management(extent_list, r"in_memory\temp") - - # Get extent of this temporary layer and zoom to its extent - desc = arcpy.Describe(r"in_memory\temp") - extent = desc.extent - - ll_geom = arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(arcpy.SpatialReference(4326)) - ur_geom = arcpy.PointGeometry(extent.upperRight, sr).projectAs(arcpy.SpatialReference(4326)) - - ll = ll_geom.centroid - ur = ur_geom.centroid - - new_sr = create_custom_spatial_ref(ll, ur) - - set_extent(aprx, extent, sr, new_sr) - - # Clean up - arcpy.Delete_management(r"in_memory\temp") - del ext, desc, array, extent_list - - # Save aprx so that after step is run user can open the aprx at the right zoom/ extent to continue examining data. - aprx.save() - - logger.info("start: building dictionaries of time steps and commodities occurring within the scenario") - commodity_dict = {} - time_dict = {} - - flds = ["COMMODITY", "TIME_PERIOD"] - - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, 'optimized_route_segments'), flds) as cursor: - for row in cursor: - if row[0] not in commodity_dict: - commodity_dict[row[0]] = True - if row[1] not in time_dict: - time_dict[row[1]] = True - - # Add Flag Fields to all of the feature classes which will need to be mapped (first delete if they already exist). - - if len(arcpy.ListFields("optimized_route_segments", "Include_Map")) > 0: - arcpy.DeleteField_management("optimized_route_segments", ["Include_Map"]) - if len(arcpy.ListFields("raw_material_producers", "Include_Map")) > 0: - arcpy.DeleteField_management("raw_material_producers", ["Include_Map"]) - if len(arcpy.ListFields("processors", "Include_Map")) > 0: - arcpy.DeleteField_management("processors", ["Include_Map"]) - if len(arcpy.ListFields("ultimate_destinations", "Include_Map")) > 0: - arcpy.DeleteField_management("ultimate_destinations", ["Include_Map"]) - - arcpy.AddField_management(os.path.join(scenario_gdb, "optimized_route_segments"), "Include_Map", "SHORT") - arcpy.AddField_management(os.path.join(scenario_gdb, "raw_material_producers"), "Include_Map", "SHORT") - arcpy.AddField_management(os.path.join(scenario_gdb, "processors"), "Include_Map", "SHORT") - arcpy.AddField_management(os.path.join(scenario_gdb, "ultimate_destinations"), "Include_Map", "SHORT") - - # Iterate through each commodity - for commodity in commodity_dict: - layer_name = "commodity_" + commodity - logger.info("Processing " + layer_name) - image_name = "optimal_flows_commodity_" + commodity + "_" + basemap - sql_where_clause = "COMMODITY = '" + commodity + "'" - - # ID route segments and facilities associated with the subset - link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) - - # Make dissolved (aggregate) fc for this commodity - dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, - the_scenario, logger) - - # Make map - make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) - - # Clear flag fields - clear_flag_fields(the_scenario) - - # Iterate through each time period - for time in time_dict: - layer_name = "time_period_" + str(time) - logger.info("Processing " + layer_name) - image_name = "optimal_flows_time_" + str(time) + "_" + basemap - sql_where_clause = "TIME_PERIOD = '" + str(time) + "'" - - # ID route segments and facilities associated with the subset - link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) - - # Make dissolved (aggregate) fc for this commodity - dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, - the_scenario, logger) - - # Make map - make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) - - # Clear flag fields - clear_flag_fields(the_scenario) - - # Iterate through each commodity/time period combination - for commodity in commodity_dict: - for time in time_dict: - layer_name = "commodity_" + commodity + "_time_period_" + str(time) - logger.info("Processing " + layer_name) - image_name = "optimal_flows_commodity_" + commodity + "_time_" + str(time) + "_" + basemap - sql_where_clause = "COMMODITY = '" + commodity + "' AND TIME_PERIOD = '" + str(time) + "'" - - # ID route segments and facilities associated with the subset - link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) - - # Make dissolved (aggregate) fc for this commodity - dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, - the_scenario, logger) - - # Make map - make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) - - # Clear flag fields - clear_flag_fields(the_scenario) - - # Create time animation out of time step maps - map_animation(the_scenario, logger) - - logger.info("time and commodity maps located here: {}".format(the_scenario.mapping_directory)) - - -# =================================================================================================== -def link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario): - - scenario_gdb = the_scenario.main_gdb - - # Create dictionaries for tracking facilities - facilities_dict = {} - segment_dict = {} - - # Initiate search cursor to ID route subset - - # get a list of the optimal facilities that share the commodity/time period of interest - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select facility_name from optimal_facilities where {};""".format(sql_where_clause) - db_cur = db_con.execute(sql) - sql_facility_data = db_cur.fetchall() - for facilities in sql_facility_data: - facilities_dict[facilities[0]] = True - # print facilities_dict - - # Flag raw_material_suppliers, processors, and destinations that are used for the subset of routes - - for facility_type in ["raw_material_producers", "processors", "ultimate_destinations"]: - with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, facility_type), - ['facility_name', 'Include_Map']) as ucursor1: - for row in ucursor1: - if row[0] in facilities_dict: - row[1] = 1 - else: - row[1] = 0 - ucursor1.updateRow(row) - - # Flag route segments that occur within the subset of routes - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "optimized_route_segments"), - ['ObjectID', 'Include_Map', 'Commodity', 'Time_Period'], sql_where_clause) as scursor1: - for row in scursor1: - segment_dict[row[0]] = True - with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, "optimized_route_segments"), - ['ObjectID', 'Include_Map', 'Commodity', 'Time_Period'],) as ucursor2: - for row in ucursor2: - if row[0] in segment_dict: - row[1] = 1 - else: - row[1] = 0 - ucursor2.updateRow(row) - - # Clean up dictionary - del facilities_dict - del segment_dict - - -# =================================================================================================== -def make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap): - - scenario_gdb = the_scenario.main_gdb - - # Check if route segment layer has any data-- - # Currently only mapping if there is route data - if arcpy.Exists("route_segments_lyr"): - arcpy.Delete_management("route_segments_lyr") - arcpy.MakeFeatureLayer_management(os.path.join(scenario_gdb, 'optimized_route_segments'), - "route_segments_lyr", "Include_Map = 1") - result = arcpy.GetCount_management("route_segments_lyr") - count = int(result.getOutput(0)) - - if count > 0: - - # reset the map so we are working from a clean and known starting point. - reset_map_base_layers(aprx, logger, basemap) - - # get a dictionary of all the layers in the aprx - # might want to get a list of groups - layer_dictionary = get_layer_dictionary(aprx, logger) - - # create a variable for for each layer of interest for the time and commodity mapping - # so we can access each layer easily - time_commodity_segments_lyr = layer_dictionary["TIME_COMMODITY"] - - # START MAKING THE MAPS! - - # Establish definition queries to define the subset for each layer, - # turn off if there are no features for that particular subset. - for groupLayer in [time_commodity_segments_lyr]: - sublayers = groupLayer.listLayers() - for sublayer in sublayers: - if sublayer.supports("DATASOURCE"): - if sublayer.dataSource == os.path.join(scenario_gdb, 'optimized_route_segments'): - sublayer.definitionQuery = "Include_Map = 1" - - if sublayer.dataSource == os.path.join(scenario_gdb, 'raw_material_producers'): - sublayer.definitionQuery = "Include_Map = 1" - rmp_count = get_feature_count(sublayer, logger) - - if sublayer.dataSource == os.path.join(scenario_gdb, 'processors'): - sublayer.definitionQuery = "Include_Map = 1" - proc_count = get_feature_count(sublayer, logger) - - if sublayer.dataSource == os.path.join(scenario_gdb, 'ultimate_destinations'): - sublayer.definitionQuery = "Include_Map = 1" - dest_count = get_feature_count(sublayer, logger) - - # Actually export map to file - time_commodity_segments_lyr.visible = True - sublayers = time_commodity_segments_lyr.listLayers() - for sublayer in sublayers: - sublayer.visible = True - caption = "" - generate_map(caption, image_name, aprx, the_scenario, logger, basemap) - - # Clean up aprx - del aprx - - # No mapping if there are no routes - else: - logger.info("no routes for this combination of time steps and commodities... skipping mapping...") - - -# =================================================================================================== -def clear_flag_fields(the_scenario): - - scenario_gdb = the_scenario.main_gdb - - # Everything is set to 1 for cleanup - # 0's are only set in link_route_to_route_segments_and_facilities, when fc's are subset by commodity/time step - arcpy.CalculateField_management(os.path.join(scenario_gdb, "optimized_route_segments"), "Include_Map", 1, - 'PYTHON_9.3') - arcpy.CalculateField_management(os.path.join(scenario_gdb, "raw_material_producers"), "Include_Map", 1, - 'PYTHON_9.3') - arcpy.CalculateField_management(os.path.join(scenario_gdb, "processors"), "Include_Map", 1, 'PYTHON_9.3') - arcpy.CalculateField_management(os.path.join(scenario_gdb, "ultimate_destinations"), "Include_Map", 1, 'PYTHON_9.3') - - -# =================================================================================================== -def map_animation(the_scenario, logger): - - # Below animation is currently only set up to animate scenario time steps - # NOT commodities or a combination of commodity and time steps. - - # Clean Up-- delete existing gif if it exists already - - try: - os.remove(os.path.join(the_scenario.mapping_directory, 'optimal_flows_time.gif')) - logger.debug("deleted existing time animation gif") - except OSError: - pass - - images = [] - - logger.info("start: creating time step animation gif") - for a_file in os.listdir(the_scenario.mapping_directory): - if a_file.startswith("optimal_flows_time"): - images.append(imageio.imread(os.path.join(the_scenario.mapping_directory, a_file))) - - if len(images) > 0: - imageio.mimsave(os.path.join(the_scenario.mapping_directory, 'optimal_flows_time.gif'), images, duration=2) - - -# =================================================================================================== -def get_feature_count(fc, logger): - result = arcpy.GetCount_management(fc) - count = int(result.getOutput(0)) - logger.debug("number of features in fc {}: \t{}".format(fc, count)) - return count - - -# =================================================================================================== -def create_custom_spatial_ref(ll, ur): - - # prevent errors by setting to default USA Contiguous Lambert Conformal Conic projection if there are problems - # basing projection off of the facilities - try: - central_meridian = str(((ur.X - ll.X) / 2) + ll.X) - except: - central_meridian = -96.0 - try: - stand_par_1 = str(((ur.Y - ll.Y) / 6) + ll.Y) - except: - stand_par_1 = 33.0 - try: - stand_par_2 = str(ur.Y - ((ur.Y - ll.Y) / 6)) - except: - stand_par_2 = 45.0 - try: - lat_orig = str(((ur.Y - ll.Y) / 2) + ll.Y) - except: - lat_orig = 39.0 - - projection = "PROJCS['Custom_Lambert_Conformal_Conic'," \ - "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]]," \ - "PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]," \ - "PROJECTION['Lambert_Conformal_Conic']," \ - "PARAMETER['False_Easting',0.0]," \ - "PARAMETER['False_Northing',0.0]," \ - "PARAMETER['Central_Meridian'," + central_meridian + "]," \ - "PARAMETER['Standard_Parallel_1'," + stand_par_1 + "]," \ - "PARAMETER['Standard_Parallel_2'," + stand_par_2 + "]," \ - "PARAMETER['Latitude_Of_Origin'," + lat_orig + "]," \ - "UNIT['Meter',1.0]]" - - new_sr = arcpy.SpatialReference() - new_sr.loadFromString(projection) - - return new_sr - - -# =================================================================================================== -def set_extent(aprx, extent, sr, new_sr): - - map = aprx.listMaps("ftot_map")[0] - map_layout = aprx.listLayouts("ftot_layout")[0] - map_frame = map_layout.listElements("MAPFRAME_ELEMENT", "FTOT")[0] - - map.spatialReference = new_sr - - ll_geom, lr_geom, ur_geom, ul_geom = [arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(new_sr), - arcpy.PointGeometry(extent.lowerRight, sr).projectAs(new_sr), - arcpy.PointGeometry(extent.upperRight, sr).projectAs(new_sr), - arcpy.PointGeometry(extent.upperLeft, sr).projectAs(new_sr)] - - ll = ll_geom.centroid - lr = lr_geom.centroid - ur = ur_geom.centroid - ul = ul_geom.centroid - - ext_buff_dist_x = ((int(abs(ll.X - lr.X))) * .15) - ext_buff_dist_y = ((int(abs(ll.Y - ul.Y))) * .15) - ext_buff_dist = max([ext_buff_dist_x, ext_buff_dist_y]) - orig_extent_pts = arcpy.Array() - # Array to hold points for the bounding box for initial extent - for coords in [ll, lr, ur, ul, ll]: - orig_extent_pts.add(coords) - - polygon_tmp_1 = arcpy.Polygon(orig_extent_pts) - # buffer the temporary poly by 10% of width or height of extent as calculated above - buff_poly = polygon_tmp_1.buffer(ext_buff_dist) - new_extent = buff_poly.extent - - map_frame.camera.setExtent(new_extent) - - -# =================================================================================================== -def dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, the_scenario, - logger): - - # Make a dissolved version of fc for mapping aggregate flows - logger.info("start: dissolve_optimal_route_segments_feature_class_for_commodity_mapping") - - scenario_gdb = the_scenario.main_gdb - - arcpy.env.workspace = scenario_gdb - - # Delete previous fcs if they exist - for fc in ["optimized_route_segments_dissolved_tmp", "optimized_route_segments_split_tmp", - "optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved_tmp2", - "dissolved_segments_lyr", "optimized_route_segments_dissolved_commodity", - "optimized_route_segments_dissolved_" + layer_name]: - if arcpy.Exists(fc): - arcpy.Delete_management(fc) - - arcpy.MakeFeatureLayer_management("optimized_route_segments", "optimized_route_segments_lyr") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="optimized_route_segments_lyr", - selection_type="NEW_SELECTION", where_clause=sql_where_clause) - - # Dissolve - arcpy.Dissolve_management("optimized_route_segments_lyr", "optimized_route_segments_dissolved_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL"], - [['COMMODITY_FLOW', 'SUM']], "SINGLE_PART", "DISSOLVE_LINES") - - # Second dissolve needed to accurately show aggregate pipeline flows - arcpy.FeatureToLine_management("optimized_route_segments_dissolved_tmp", "optimized_route_segments_split_tmp") - - arcpy.AddGeometryAttributes_management("optimized_route_segments_split_tmp", "LINE_START_MID_END") - - arcpy.Dissolve_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2", - ["NET_SOURCE_NAME", "Shape_Length", "MID_X", "MID_Y", "ARTIFICIAL"], - [["SUM_COMMODITY_FLOW", "SUM"]], "SINGLE_PART", "DISSOLVE_LINES") - - arcpy.AddField_management(in_table="optimized_route_segments_dissolved_tmp2", field_name="SUM_COMMODITY_FLOW", - field_type="DOUBLE", field_precision="", field_scale="", field_length="", field_alias="", - field_is_nullable="NULLABLE", field_is_required="NON_REQUIRED", field_domain="") - arcpy.CalculateField_management(in_table="optimized_route_segments_dissolved_tmp2", field="SUM_COMMODITY_FLOW", - expression="!SUM_SUM_COMMODITY_FLOW!", expression_type="PYTHON_9.3", code_block="") - arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", - drop_field="SUM_SUM_COMMODITY_FLOW") - arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", drop_field="MID_X") - arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", drop_field="MID_Y") - - # Sort for mapping order - arcpy.AddField_management(in_table="optimized_route_segments_dissolved_tmp2", field_name="SORT_FIELD", - field_type="SHORT") - arcpy.MakeFeatureLayer_management("optimized_route_segments_dissolved_tmp2", "dissolved_segments_lyr") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", - where_clause="NET_SOURCE_NAME = 'road'") - arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", - expression=1, expression_type="PYTHON_9.3") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", - where_clause="NET_SOURCE_NAME = 'rail'") - arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", - expression=2, expression_type="PYTHON_9.3") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", - where_clause="NET_SOURCE_NAME = 'water'") - arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", - expression=3, expression_type="PYTHON_9.3") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", - where_clause="NET_SOURCE_NAME LIKE 'pipeline%'") - arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", - expression=4, expression_type="PYTHON_9.3") - - arcpy.Sort_management("optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved_commodity", - [["SORT_FIELD", "ASCENDING"]]) - - # Delete temp fc's - arcpy.Delete_management("optimized_route_segments_dissolved_tmp") - arcpy.Delete_management("optimized_route_segments_split_tmp") - arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") - arcpy.Delete_management("optimized_route_segments_lyr") - arcpy.Delete_management("dissolved_segments_lyr") - - # Copy to permanent fc (unique to commodity name) - arcpy.CopyFeatures_management("optimized_route_segments_dissolved_commodity", - "optimized_route_segments_dissolved_" + layer_name) + +# --------------------------------------------------------------------------------------------------- +# Name: ftot_maps +# --------------------------------------------------------------------------------------------------- +import os +import arcpy +from shutil import copy +import imageio +import sqlite3 +import datetime +from six import iteritems + + +# =================================================================================================== +def new_map_creation(the_scenario, logger, task): + + logger.info("start: maps") + + # create map directory + timestamp_folder_name = 'maps_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + + if task == "m": + basemap = "default_basemap" + elif task == "mb": + basemap = "gray_basemap" + elif task == "mc": + basemap = "topo_basemap" + elif task == "md": + basemap = "street_basemap" + else: + basemap = "default_basemap" + + the_scenario.mapping_directory = os.path.join(the_scenario.scenario_run_directory, "Maps", + basemap + "_" + timestamp_folder_name) + scenario_gdb = the_scenario.main_gdb + + scenario_aprx_location = os.path.join(the_scenario.scenario_run_directory, "Maps", "ftot_maps.aprx") + if not os.path.exists(the_scenario.mapping_directory): + logger.debug("creating maps directory.") + os.makedirs(the_scenario.mapping_directory) + + # Must delete existing scenario aprx because there can only be one per scenario (everything in the aprx references + # the existing scenario main.gdb) + if os.path.exists(scenario_aprx_location): + os.remove(scenario_aprx_location) + + # set aprx locations: root is in the common data folder, and the scenario is in the local maps folder + root_ftot_aprx_location = os.path.join(the_scenario.common_data_folder, "ftot_maps.aprx") + base_layers_location = os.path.join(the_scenario.common_data_folder, "Base_Layers") + + # copy the aprx from the common data director to the scenario maps directory + logger.debug("copying the root aprx from common data to the scenario maps directory.") + copy(root_ftot_aprx_location, scenario_aprx_location) + + # load the map document into memory + aprx = arcpy.mp.ArcGISProject(scenario_aprx_location) + + # Fix the non-base layers here + broken_list = aprx.listBrokenDataSources() + for broken_item in broken_list: + if broken_item.supports("DATASOURCE"): + if not broken_item.longName.find("Base") == 0: + conprop = broken_item.connectionProperties + conprop['connection_info']['database'] = scenario_gdb + broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) + + list_broken_data_sources(aprx, base_layers_location, logger) + + reset_map_base_layers(aprx, logger, basemap) + + export_map_steps(aprx, the_scenario, logger, basemap) + + logger.info("maps located here: {}".format(the_scenario.mapping_directory)) + + +# =================================================================================================== +def list_broken_data_sources(aprx, base_layers_location, logger): + broken_list = aprx.listBrokenDataSources() + for broken_item in broken_list: + if broken_item.supports("DATASOURCE"): + if broken_item.longName.find("Base") == 0: + conprop = broken_item.connectionProperties + conprop['connection_info']['database'] = base_layers_location + broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) + logger.debug("\t" + "broken base layer path fixed: " + broken_item.name) + else: + logger.debug("\t" + "broken aprx data source path: " + broken_item.name) + + +# =================================================================================================== +def reset_map_base_layers(aprx, logger, basemap): + + logger.debug("start: reset_map_base_layers") + # turn on base layers, turn off everything else. + map = aprx.listMaps("ftot_map")[0] + for lyr in map.listLayers(): + + if lyr.longName.find("Base") == 0: + lyr.visible = True + + # This group layer should be off unless mb step is being run + if basemap not in ["gray_basemap"] and "World Light Gray" in lyr.longName and \ + not lyr.longName.endswith("Light Gray Canvas Base"): + lyr.visible = False + + # This group layer should be off unless mc step is being run + if basemap not in ["topo_basemap"] and "USGSTopo" in lyr.longName and not lyr.longName.endswith("Layers"): + lyr.visible = False + + # This group layer should be off unless md step is being run + if basemap not in ["street_basemap"] and "World Street Map" in lyr.longName and not lyr.longName.endswith("Layers"): + lyr.visible = False + + # These layers can't be turned off. So leave them on. + elif lyr.longName.endswith("Light Gray Canvas Base"): + lyr.visible = True + elif lyr.longName == "Layers": + lyr.visible = True + + else: + lyr.visible = False + + layout = aprx.listLayouts("ftot_layout")[0] + elm = layout.listElements("TEXT_ELEMENT")[0] + elm.text = " " + + return + + +# =================================================================================================== +def get_layer_dictionary(aprx, logger): + logger.debug("start: get_layer_dictionary") + layer_dictionary = {} + map = aprx.listMaps("ftot_map")[0] + for lyr in map.listLayers(): + + layer_dictionary[lyr.longName.replace(" ", "_").replace("\\", "_")] = lyr + + return layer_dictionary + + +# =================================================================================================== +def debug_layer_status(aprx, logger): + + layer_dictionary = get_layer_dictionary(aprx, logger) + + for layer_name, lyr in sorted(iteritems(layer_dictionary)): + + logger.info("layer: {}, visible: {}".format(layer_name, lyr.visible)) + + +# ===================================================================================================- +def export_to_png(map_name, aprx, the_scenario, logger): + + file_name = str(map_name + ".png").replace(" ", "_").replace("\\", "_") + + file_location = os.path.join(the_scenario.mapping_directory, file_name) + + layout = aprx.listLayouts("ftot_layout")[0] + + # # Setup legend properties so everything refreshes as expected + legend = layout.listElements("LEGEND_ELEMENT", "Legend")[0] + legend.fittingStrategy = 'AdjustFrame' + legend.syncLayerVisibility = True + legend.syncLayerOrder = True + aprx.save() + + layout.exportToPNG(file_location) + + logger.info("exporting: {}".format(map_name)) + + +# =================================================================================================== +def export_map_steps(aprx, the_scenario, logger, basemap): + + # ------------------------------------------------------------------------------------ + + # Project and zoom to extent of features + # Get extent of the locations and optimized_route_segments FCs and zoom to buffered extent + + # A list of extents + extent_list = [] + + for fc in [os.path.join(the_scenario.main_gdb, "locations"), + os.path.join(the_scenario.main_gdb, "optimized_route_segments")]: + # Cycle through layers grabbing extents, converting them into + # polygons and adding them to extentList + desc = arcpy.Describe(fc) + ext = desc.extent + array = arcpy.Array([ext.upperLeft, ext.upperRight, ext.lowerRight, ext.lowerLeft]) + extent_list.append(arcpy.Polygon(array)) + sr = desc.spatialReference + + # Create a temporary FeatureClass from the polygons + arcpy.CopyFeatures_management(extent_list, r"in_memory\temp") + + # Get extent of this temporary layer and zoom to its extent + desc = arcpy.Describe(r"in_memory\temp") + extent = desc.extent + + ll_geom = arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(arcpy.SpatialReference(4326)) + ur_geom = arcpy.PointGeometry(extent.upperRight, sr).projectAs(arcpy.SpatialReference(4326)) + + ll = ll_geom.centroid + ur = ur_geom.centroid + + new_sr = create_custom_spatial_ref(ll, ur) + + set_extent(aprx, extent, sr, new_sr) + + # Clean up + arcpy.Delete_management(r"in_memory\temp") + del ext, desc, array, extent_list + + # Save aprx so that after step is run user can open the aprx at the right zoom/ extent to continue examining the data. + aprx.save() + + # reset the map so we are working from a clean and known starting point. + reset_map_base_layers(aprx, logger, basemap) + + # get a dictionary of all the layers in the aprx + # might want to get a list of groups + layer_dictionary = get_layer_dictionary(aprx, logger) + + logger.debug("layer_dictionary.keys(): \t {}".format(list(layer_dictionary.keys()))) + + # create a variable for each layer so we can access each layer easily + + s_step_lyr = layer_dictionary["S_STEP"] + f_step_rmp_lyr = layer_dictionary["F_STEP_RMP"] + f_step_proc_lyr = layer_dictionary["F_STEP_PROC"] + f_step_dest_lyr = layer_dictionary["F_STEP_DEST"] + f_step_lyr = layer_dictionary["F_STEP"] + f_step_rmp_labels_lyr = layer_dictionary["F_STEP_RMP_W_LABELS"] + f_step_proc_labels_lyr = layer_dictionary["F_STEP_PROC_W_LABELS"] + f_step_dest_labels_lyr = layer_dictionary["F_STEP_DEST_W_LABELS"] + f_step_labels_lyr = layer_dictionary["F_STEP_W_LABELS"] + o_step_opt_flow_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW"] + o_step_opt_flow_no_labels_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW_NO_LABELS"] + o_step_opt_flow_just_flow_lyr = layer_dictionary["O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW_JUST_FLOW"] + o_step_rmp_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_RMP_OPTIMAL_VS_NON_OPTIMAL"] + o_step_proc_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_PROC_OPTIMAL_VS_NON_OPTIMAL"] + o_step_dest_opt_vs_non_opt_lyr = layer_dictionary["O_STEP_DEST_OPTIMAL_VS_NON_OPTIMAL"] + o_step_raw_producer_to_processor_lyr = layer_dictionary["O_STEP_RMP_TO_PROC"] + o_step_processor_to_destination_lyr = layer_dictionary["O_STEP_PROC_TO_DEST"] + f2_step_candidates_lyr = layer_dictionary["F2_STEP_CANDIDATES"] + f2_step_merged_lyr = layer_dictionary["F2_STEP_MERGE"] + f2_step_candidates_labels_lyr = layer_dictionary["F2_STEP_CANDIDATES_W_LABELS"] + f2_step_merged_labels_lyr = layer_dictionary["F2_STEP_MERGE_W_LABELS"] + + # Custom user-created maps-- user can create group layers within this parent group layer containing different + # features they would like to map. Code will look for this group layer name, which should not change. + if "CUSTOM_USER_CREATED_MAPS" in layer_dictionary: + custom_maps_parent_lyr = layer_dictionary["CUSTOM_USER_CREATED_MAPS"] + else: + custom_maps_parent_lyr = None + + # START MAKING THE MAPS! + +# turn off all the groups +# turn on the group step +# set caption information +# Optimal Processor to Optimal Ultimate Destination Delivery Routes +# opt_destinations_lyr.visible = True +# map_name = "" +# caption = "" +# call generate_map() +# generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # S_STEP + s_step_lyr.visible = True + sublayers = s_step_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "01_S_Step_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_RMP + f_step_rmp_lyr.visible = True + sublayers = f_step_rmp_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02a_F_Step_RMP_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_PROC + f_step_proc_lyr.visible = True + sublayers = f_step_proc_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02b_F_Step_User_Defined_PROC_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_DEST + f_step_dest_lyr.visible = True + sublayers = f_step_dest_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02c_F_Step_DEST_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP + f_step_lyr.visible = True + sublayers = f_step_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02d_F_Step_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_RMP_LABELS + f_step_rmp_labels_lyr.visible = True + sublayers = f_step_rmp_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02a_F_Step_RMP_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_PROC_LABELS + f_step_proc_labels_lyr.visible = True + sublayers = f_step_proc_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02b_F_Step_User_Defined_PROC_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_DEST_LABELS + f_step_dest_labels_lyr.visible = True + sublayers = f_step_dest_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02c_F_Step_DEST_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F_STEP_LABELS + f_step_labels_lyr.visible = True + sublayers = f_step_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "02d_F_Step_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_FINAL_OPTIMAL_ROUTES_WITH_COMMODITY_FLOW + # O_STEP - A - + o_step_opt_flow_lyr.visible = True + sublayers = o_step_opt_flow_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04a_O_Step_Final_Optimal_Routes_With_Commodity_Flow_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP - B - same as above but with no labels to make it easier to see routes. + o_step_opt_flow_no_labels_lyr.visible = True + sublayers = o_step_opt_flow_no_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04b_O_Step_Final_Optimal_Routes_With_Commodity_Flow_NO_LABELS_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP - C - just flows. no Origins or Destinations + o_step_opt_flow_just_flow_lyr.visible = True + sublayers = o_step_opt_flow_just_flow_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04c_O_Step_Final_Optimal_Routes_With_Commodity_Flow_JUST_FLOW_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_OPTIMAL_AND_NON_OPTIMAL_RMP + o_step_rmp_opt_vs_non_opt_lyr.visible = True + sublayers = o_step_rmp_opt_vs_non_opt_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04d_O_Step_Optimal_and_Non_Optimal_RMP_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_OPTIMAL_AND_NON_OPTIMAL_DEST + o_step_proc_opt_vs_non_opt_lyr.visible = True + sublayers = o_step_proc_opt_vs_non_opt_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04e_O_Step_Optimal_and_Non_Optimal_PROC_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_OPTIMAL_AND_NON_OPTIMAL_DEST + o_step_dest_opt_vs_non_opt_lyr.visible = True + sublayers = o_step_dest_opt_vs_non_opt_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "04f_O_Step_Optimal_and_Non_Optimal_DEST_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # check if processor fc exists. if it doesn't stop here since there won't be any more relevant maps to make + + processor_merged_fc = the_scenario.processors_fc + processor_fc_feature_count = get_feature_count(processor_merged_fc, logger) + + if processor_fc_feature_count: + logger.debug("Processor Feature Class has {} features....".format(processor_fc_feature_count)) + # F2_STEP_CANDIDATES + f2_step_candidates_lyr.visible = True + sublayers = f2_step_candidates_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "03a_F2_Step_Processor_Candidates_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F2_STEP_MERGE + f2_step_merged_lyr.visible = True + sublayers = f2_step_merged_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "03b_F2_Step_Processors_All_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F2_STEP_CANDIDATES_W_LABELS + f2_step_candidates_labels_lyr.visible = True + sublayers = f2_step_candidates_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "03a_F2_Step_Processor_Candidates_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # F2_STEP_MERGE_W_LABELS + f2_step_merged_labels_lyr.visible = True + sublayers = f2_step_merged_labels_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "03b_F2_Step_Processors_All_With_Labels_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_RAW_MATERIAL_PRODUCER_TO_PROCESSOR + o_step_raw_producer_to_processor_lyr.visible = True + sublayers = o_step_raw_producer_to_processor_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "05a_O_Step_Raw_Material_Producer_To_Processor_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + # O_STEP_PROCESSOR_TO_ULTIMATE_DESTINATION + o_step_processor_to_destination_lyr.visible = True + sublayers = o_step_processor_to_destination_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + map_name = "05b_O_Step_Processor_To_Ultimate_Destination_" + basemap + caption = "" + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + else: + logger.info("processor layer has {} features. skipping the rest of the mapping".format + (processor_fc_feature_count)) + + # CUSTOM_MAPS + if "CUSTOM_USER_CREATED_MAPS" in layer_dictionary: + sub_groups_custom_maps = custom_maps_parent_lyr.listLayers() + for sub_group in sub_groups_custom_maps: + # Need to limit maps to just the group layers which are NOT the parent group layer. + if sub_group.isGroupLayer and sub_group != custom_maps_parent_lyr: + # First ensure that the group layer is on as this as all layers are turned off at beginning by default + custom_maps_parent_lyr.visible = True + # Then turn on the individual custom map + sub_group.visible = True + count = 0 + sublayers = sub_group.listLayers() + for sublayer in sublayers: + sublayer.visible = True + count += 1 + map_name = str(sub_group) + "_" + basemap + caption = "" + # Only generate map if there are actual features inside the group layer + if count > 0: + generate_map(caption, map_name, aprx, the_scenario, logger, basemap) + + +# =================================================================================================== +def generate_map(caption, map_name, aprx, the_scenario, logger, basemap): + import datetime + + # create a text element on the aprx for the caption + layout = aprx.listLayouts("ftot_layout")[0] + elm = layout.listElements("TEXT_ELEMENT")[0] + + # set the caption text + dt = datetime.datetime.now() + + elm.text = "Scenario Name: " + the_scenario.scenario_name + " -- Date: " + str(dt.strftime("%b %d, %Y")) + "\n" + \ + caption + + # programmatically set the caption height + if len(caption) < 180: + elm.elementHeight = 0.5 + + else: + elm.elementHeight = 0.0017*len(caption) + 0.0 + + # export that map to png + export_to_png(map_name, aprx, the_scenario, logger) + + # reset the map layers + reset_map_base_layers(aprx, logger, basemap) + + +# =================================================================================================== +def prepare_time_commodity_subsets_for_mapping(the_scenario, logger, task): + + logger.info("start: time and commodity maps") + + if task == "m2": + basemap = "default_basemap" + elif task == "m2b": + basemap = "gray_basemap" + elif task == "m2c": + basemap = "topo_basemap" + elif task == "m2d": + basemap = "street_basemap" + else: + basemap = "default_basemap" + + timestamp_folder_name = 'maps_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + the_scenario.mapping_directory = os.path.join(the_scenario.scenario_run_directory, "Maps_Time_Commodity", + basemap + "_" + timestamp_folder_name) + scenario_aprx_location = os.path.join(the_scenario.scenario_run_directory, "Maps_Time_Commodity", "ftot_maps.aprx") + scenario_gdb = the_scenario.main_gdb + arcpy.env.workspace = scenario_gdb + + if not os.path.exists(the_scenario.mapping_directory): + logger.debug("creating maps directory.") + os.makedirs(the_scenario.mapping_directory) + + if os.path.exists(scenario_aprx_location): + os.remove(scenario_aprx_location) + + # set aprx locations: root is in the common data folder, and the scenario is in the local maps folder + root_ftot_aprx_location = os.path.join(the_scenario.common_data_folder, "ftot_maps.aprx") + base_layers_location = os.path.join(the_scenario.common_data_folder, "Base_Layers") + + # copy the aprx from the common data director to the scenario maps directory + logger.debug("copying the root aprx from common data to the scenario maps directory.") + copy(root_ftot_aprx_location, scenario_aprx_location) + + # load the map document into memory + aprx = arcpy.mp.ArcGISProject(scenario_aprx_location) + + # Fix the non-base layers here + broken_list = aprx.listBrokenDataSources() + for broken_item in broken_list: + if broken_item.supports("DATASOURCE"): + if not broken_item.longName.find("Base") == 0: + conprop = broken_item.connectionProperties + conprop['connection_info']['database'] = scenario_gdb + broken_item.updateConnectionProperties(broken_item.connectionProperties, conprop) + + list_broken_data_sources(aprx, base_layers_location, logger) + + # Project and zoom to extent of features + # Get extent of the locations and optimized_route_segments FCs and zoom to buffered extent + + # A list of extents + extent_list = [] + + for fc in [os.path.join(the_scenario.main_gdb, "locations"), + os.path.join(the_scenario.main_gdb, "optimized_route_segments")]: + # Cycle through layers grabbing extents, converting them into + # polygons and adding them to extentList + desc = arcpy.Describe(fc) + ext = desc.extent + array = arcpy.Array([ext.upperLeft, ext.upperRight, ext.lowerRight, ext.lowerLeft]) + extent_list.append(arcpy.Polygon(array)) + sr = desc.spatialReference + + # Create a temporary FeatureClass from the polygons + arcpy.CopyFeatures_management(extent_list, r"in_memory\temp") + + # Get extent of this temporary layer and zoom to its extent + desc = arcpy.Describe(r"in_memory\temp") + extent = desc.extent + + ll_geom = arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(arcpy.SpatialReference(4326)) + ur_geom = arcpy.PointGeometry(extent.upperRight, sr).projectAs(arcpy.SpatialReference(4326)) + + ll = ll_geom.centroid + ur = ur_geom.centroid + + new_sr = create_custom_spatial_ref(ll, ur) + + set_extent(aprx, extent, sr, new_sr) + + # Clean up + arcpy.Delete_management(r"in_memory\temp") + del ext, desc, array, extent_list + + # Save aprx so that after step is run user can open the aprx at the right zoom/ extent to continue examining data. + aprx.save() + + logger.info("start: building dictionaries of time steps and commodities occurring within the scenario") + commodity_dict = {} + time_dict = {} + + flds = ["COMMODITY", "TIME_PERIOD"] + + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, 'optimized_route_segments'), flds) as cursor: + for row in cursor: + if row[0] not in commodity_dict: + commodity_dict[row[0]] = True + if row[1] not in time_dict: + time_dict[row[1]] = True + + # Add Flag Fields to all of the feature classes which will need to be mapped (first delete if they already exist). + + if len(arcpy.ListFields("optimized_route_segments", "Include_Map")) > 0: + arcpy.DeleteField_management("optimized_route_segments", ["Include_Map"]) + if len(arcpy.ListFields("raw_material_producers", "Include_Map")) > 0: + arcpy.DeleteField_management("raw_material_producers", ["Include_Map"]) + if len(arcpy.ListFields("processors", "Include_Map")) > 0: + arcpy.DeleteField_management("processors", ["Include_Map"]) + if len(arcpy.ListFields("ultimate_destinations", "Include_Map")) > 0: + arcpy.DeleteField_management("ultimate_destinations", ["Include_Map"]) + + arcpy.AddField_management(os.path.join(scenario_gdb, "optimized_route_segments"), "Include_Map", "SHORT") + arcpy.AddField_management(os.path.join(scenario_gdb, "raw_material_producers"), "Include_Map", "SHORT") + arcpy.AddField_management(os.path.join(scenario_gdb, "processors"), "Include_Map", "SHORT") + arcpy.AddField_management(os.path.join(scenario_gdb, "ultimate_destinations"), "Include_Map", "SHORT") + + # Iterate through each commodity + for commodity in commodity_dict: + layer_name = "commodity_" + commodity + logger.info("Processing " + layer_name) + image_name = "optimal_flows_commodity_" + commodity + "_" + basemap + sql_where_clause = "COMMODITY = '" + commodity + "'" + + # ID route segments and facilities associated with the subset + link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) + + # Make dissolved (aggregate) fc for this commodity + dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, + the_scenario, logger) + + # Make map + make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) + + # Clear flag fields + clear_flag_fields(the_scenario) + + # Iterate through each time period + for time in time_dict: + layer_name = "time_period_" + str(time) + logger.info("Processing " + layer_name) + image_name = "optimal_flows_time_" + str(time) + "_" + basemap + sql_where_clause = "TIME_PERIOD = '" + str(time) + "'" + + # ID route segments and facilities associated with the subset + link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) + + # Make dissolved (aggregate) fc for this commodity + dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, + the_scenario, logger) + + # Make map + make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) + + # Clear flag fields + clear_flag_fields(the_scenario) + + # Iterate through each commodity/time period combination + for commodity in commodity_dict: + for time in time_dict: + layer_name = "commodity_" + commodity + "_time_period_" + str(time) + logger.info("Processing " + layer_name) + image_name = "optimal_flows_commodity_" + commodity + "_time_" + str(time) + "_" + basemap + sql_where_clause = "COMMODITY = '" + commodity + "' AND TIME_PERIOD = '" + str(time) + "'" + + # ID route segments and facilities associated with the subset + link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario) + + # Make dissolved (aggregate) fc for this commodity + dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, + the_scenario, logger) + + # Make map + make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap) + + # Clear flag fields + clear_flag_fields(the_scenario) + + # Create time animation out of time step maps + map_animation(the_scenario, logger) + + logger.info("time and commodity maps located here: {}".format(the_scenario.mapping_directory)) + + +# =================================================================================================== +def link_subset_to_route_segments_and_facilities(sql_where_clause, the_scenario): + + scenario_gdb = the_scenario.main_gdb + + # Create dictionaries for tracking facilities + facilities_dict = {} + segment_dict = {} + + # Initiate search cursor to ID route subset + + # get a list of the optimal facilities that share the commodity/time period of interest + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select facility_name from optimal_facilities where {};""".format(sql_where_clause) + db_cur = db_con.execute(sql) + sql_facility_data = db_cur.fetchall() + for facilities in sql_facility_data: + facilities_dict[facilities[0]] = True + # print facilities_dict + + # Flag raw_material_suppliers, processors, and destinations that are used for the subset of routes + + for facility_type in ["raw_material_producers", "processors", "ultimate_destinations"]: + with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, facility_type), + ['facility_name', 'Include_Map']) as ucursor1: + for row in ucursor1: + if row[0] in facilities_dict: + row[1] = 1 + else: + row[1] = 0 + ucursor1.updateRow(row) + + # Flag route segments that occur within the subset of routes + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "optimized_route_segments"), + ['ObjectID', 'Include_Map', 'Commodity', 'Time_Period'], sql_where_clause) as scursor1: + for row in scursor1: + segment_dict[row[0]] = True + with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, "optimized_route_segments"), + ['ObjectID', 'Include_Map', 'Commodity', 'Time_Period'],) as ucursor2: + for row in ucursor2: + if row[0] in segment_dict: + row[1] = 1 + else: + row[1] = 0 + ucursor2.updateRow(row) + + # Clean up dictionary + del facilities_dict + del segment_dict + + +# =================================================================================================== +def make_time_commodity_maps(aprx, image_name, the_scenario, logger, basemap): + + scenario_gdb = the_scenario.main_gdb + + # Check if route segment layer has any data-- + # Currently only mapping if there is route data + if arcpy.Exists("route_segments_lyr"): + arcpy.Delete_management("route_segments_lyr") + arcpy.MakeFeatureLayer_management(os.path.join(scenario_gdb, 'optimized_route_segments'), + "route_segments_lyr", "Include_Map = 1") + result = arcpy.GetCount_management("route_segments_lyr") + count = int(result.getOutput(0)) + + if count > 0: + + # reset the map so we are working from a clean and known starting point. + reset_map_base_layers(aprx, logger, basemap) + + # get a dictionary of all the layers in the aprx + # might want to get a list of groups + layer_dictionary = get_layer_dictionary(aprx, logger) + + # create a variable for for each layer of interest for the time and commodity mapping + # so we can access each layer easily + time_commodity_segments_lyr = layer_dictionary["TIME_COMMODITY"] + + # START MAKING THE MAPS! + + # Establish definition queries to define the subset for each layer, + # turn off if there are no features for that particular subset. + for groupLayer in [time_commodity_segments_lyr]: + sublayers = groupLayer.listLayers() + for sublayer in sublayers: + if sublayer.supports("DATASOURCE"): + if sublayer.dataSource == os.path.join(scenario_gdb, 'optimized_route_segments'): + sublayer.definitionQuery = "Include_Map = 1" + + if sublayer.dataSource == os.path.join(scenario_gdb, 'raw_material_producers'): + sublayer.definitionQuery = "Include_Map = 1" + rmp_count = get_feature_count(sublayer, logger) + + if sublayer.dataSource == os.path.join(scenario_gdb, 'processors'): + sublayer.definitionQuery = "Include_Map = 1" + proc_count = get_feature_count(sublayer, logger) + + if sublayer.dataSource == os.path.join(scenario_gdb, 'ultimate_destinations'): + sublayer.definitionQuery = "Include_Map = 1" + dest_count = get_feature_count(sublayer, logger) + + # Actually export map to file + time_commodity_segments_lyr.visible = True + sublayers = time_commodity_segments_lyr.listLayers() + for sublayer in sublayers: + sublayer.visible = True + caption = "" + generate_map(caption, image_name, aprx, the_scenario, logger, basemap) + + # Clean up aprx + del aprx + + # No mapping if there are no routes + else: + logger.info("no routes for this combination of time steps and commodities... skipping mapping...") + + +# =================================================================================================== +def clear_flag_fields(the_scenario): + + scenario_gdb = the_scenario.main_gdb + + # Everything is set to 1 for cleanup + # 0's are only set in link_route_to_route_segments_and_facilities, when fc's are subset by commodity/time step + arcpy.CalculateField_management(os.path.join(scenario_gdb, "optimized_route_segments"), "Include_Map", 1, + 'PYTHON_9.3') + arcpy.CalculateField_management(os.path.join(scenario_gdb, "raw_material_producers"), "Include_Map", 1, + 'PYTHON_9.3') + arcpy.CalculateField_management(os.path.join(scenario_gdb, "processors"), "Include_Map", 1, 'PYTHON_9.3') + arcpy.CalculateField_management(os.path.join(scenario_gdb, "ultimate_destinations"), "Include_Map", 1, 'PYTHON_9.3') + + +# =================================================================================================== +def map_animation(the_scenario, logger): + + # Below animation is currently only set up to animate scenario time steps + # NOT commodities or a combination of commodity and time steps. + + # Clean Up-- delete existing gif if it exists already + + try: + os.remove(os.path.join(the_scenario.mapping_directory, 'optimal_flows_time.gif')) + logger.debug("deleted existing time animation gif") + except OSError: + pass + + images = [] + + logger.info("start: creating time step animation gif") + for a_file in os.listdir(the_scenario.mapping_directory): + if a_file.startswith("optimal_flows_time"): + images.append(imageio.imread(os.path.join(the_scenario.mapping_directory, a_file))) + + if len(images) > 0: + imageio.mimsave(os.path.join(the_scenario.mapping_directory, 'optimal_flows_time.gif'), images, duration=2) + + +# =================================================================================================== +def get_feature_count(fc, logger): + result = arcpy.GetCount_management(fc) + count = int(result.getOutput(0)) + logger.debug("number of features in fc {}: \t{}".format(fc, count)) + return count + + +# =================================================================================================== +def create_custom_spatial_ref(ll, ur): + + # prevent errors by setting to default USA Contiguous Lambert Conformal Conic projection if there are problems + # basing projection off of the facilities + try: + central_meridian = str(((ur.X - ll.X) / 2) + ll.X) + except: + central_meridian = -96.0 + try: + stand_par_1 = str(((ur.Y - ll.Y) / 6) + ll.Y) + except: + stand_par_1 = 33.0 + try: + stand_par_2 = str(ur.Y - ((ur.Y - ll.Y) / 6)) + except: + stand_par_2 = 45.0 + try: + lat_orig = str(((ur.Y - ll.Y) / 2) + ll.Y) + except: + lat_orig = 39.0 + + projection = "PROJCS['Custom_Lambert_Conformal_Conic'," \ + "GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]]," \ + "PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]," \ + "PROJECTION['Lambert_Conformal_Conic']," \ + "PARAMETER['False_Easting',0.0]," \ + "PARAMETER['False_Northing',0.0]," \ + "PARAMETER['Central_Meridian'," + central_meridian + "]," \ + "PARAMETER['Standard_Parallel_1'," + stand_par_1 + "]," \ + "PARAMETER['Standard_Parallel_2'," + stand_par_2 + "]," \ + "PARAMETER['Latitude_Of_Origin'," + lat_orig + "]," \ + "UNIT['Meter',1.0]]" + + new_sr = arcpy.SpatialReference() + new_sr.loadFromString(projection) + + return new_sr + + +# =================================================================================================== +def set_extent(aprx, extent, sr, new_sr): + + map = aprx.listMaps("ftot_map")[0] + map_layout = aprx.listLayouts("ftot_layout")[0] + map_frame = map_layout.listElements("MAPFRAME_ELEMENT", "FTOT")[0] + + map.spatialReference = new_sr + + ll_geom, lr_geom, ur_geom, ul_geom = [arcpy.PointGeometry(extent.lowerLeft, sr).projectAs(new_sr), + arcpy.PointGeometry(extent.lowerRight, sr).projectAs(new_sr), + arcpy.PointGeometry(extent.upperRight, sr).projectAs(new_sr), + arcpy.PointGeometry(extent.upperLeft, sr).projectAs(new_sr)] + + ll = ll_geom.centroid + lr = lr_geom.centroid + ur = ur_geom.centroid + ul = ul_geom.centroid + + ext_buff_dist_x = ((int(abs(ll.X - lr.X))) * .15) + ext_buff_dist_y = ((int(abs(ll.Y - ul.Y))) * .15) + ext_buff_dist = max([ext_buff_dist_x, ext_buff_dist_y]) + orig_extent_pts = arcpy.Array() + # Array to hold points for the bounding box for initial extent + for coords in [ll, lr, ur, ul, ll]: + orig_extent_pts.add(coords) + + polygon_tmp_1 = arcpy.Polygon(orig_extent_pts) + # buffer the temporary poly by 10% of width or height of extent as calculated above + buff_poly = polygon_tmp_1.buffer(ext_buff_dist) + new_extent = buff_poly.extent + + map_frame.camera.setExtent(new_extent) + + +# =================================================================================================== +def dissolve_optimal_route_segments_feature_class_for_commodity_mapping(layer_name, sql_where_clause, the_scenario, + logger): + + # Make a dissolved version of fc for mapping aggregate flows + logger.info("start: dissolve_optimal_route_segments_feature_class_for_commodity_mapping") + + scenario_gdb = the_scenario.main_gdb + + arcpy.env.workspace = scenario_gdb + + # Delete previous fcs if they exist + for fc in ["optimized_route_segments_dissolved_tmp", "optimized_route_segments_split_tmp", + "optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved_tmp2", + "dissolved_segments_lyr", "optimized_route_segments_dissolved_commodity", + "optimized_route_segments_dissolved_" + layer_name]: + if arcpy.Exists(fc): + arcpy.Delete_management(fc) + + arcpy.MakeFeatureLayer_management("optimized_route_segments", "optimized_route_segments_lyr") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="optimized_route_segments_lyr", + selection_type="NEW_SELECTION", where_clause=sql_where_clause) + + # Dissolve + arcpy.Dissolve_management("optimized_route_segments_lyr", "optimized_route_segments_dissolved_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL"], + [['COMMODITY_FLOW', 'SUM']], "SINGLE_PART", "DISSOLVE_LINES") + + # Second dissolve needed to accurately show aggregate pipeline flows + arcpy.FeatureToLine_management("optimized_route_segments_dissolved_tmp", "optimized_route_segments_split_tmp") + + arcpy.AddGeometryAttributes_management("optimized_route_segments_split_tmp", "LINE_START_MID_END") + + arcpy.Dissolve_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2", + ["NET_SOURCE_NAME", "Shape_Length", "MID_X", "MID_Y", "ARTIFICIAL"], + [["SUM_COMMODITY_FLOW", "SUM"]], "SINGLE_PART", "DISSOLVE_LINES") + + arcpy.AddField_management(in_table="optimized_route_segments_dissolved_tmp2", field_name="SUM_COMMODITY_FLOW", + field_type="DOUBLE", field_precision="", field_scale="", field_length="", field_alias="", + field_is_nullable="NULLABLE", field_is_required="NON_REQUIRED", field_domain="") + arcpy.CalculateField_management(in_table="optimized_route_segments_dissolved_tmp2", field="SUM_COMMODITY_FLOW", + expression="!SUM_SUM_COMMODITY_FLOW!", expression_type="PYTHON_9.3", code_block="") + arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", + drop_field="SUM_SUM_COMMODITY_FLOW") + arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", drop_field="MID_X") + arcpy.DeleteField_management(in_table="optimized_route_segments_dissolved_tmp2", drop_field="MID_Y") + + # Sort for mapping order + arcpy.AddField_management(in_table="optimized_route_segments_dissolved_tmp2", field_name="SORT_FIELD", + field_type="SHORT") + arcpy.MakeFeatureLayer_management("optimized_route_segments_dissolved_tmp2", "dissolved_segments_lyr") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", + where_clause="NET_SOURCE_NAME = 'road'") + arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", + expression=1, expression_type="PYTHON_9.3") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", + where_clause="NET_SOURCE_NAME = 'rail'") + arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", + expression=2, expression_type="PYTHON_9.3") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", + where_clause="NET_SOURCE_NAME = 'water'") + arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", + expression=3, expression_type="PYTHON_9.3") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="dissolved_segments_lyr", selection_type="NEW_SELECTION", + where_clause="NET_SOURCE_NAME LIKE 'pipeline%'") + arcpy.CalculateField_management(in_table="dissolved_segments_lyr", field="SORT_FIELD", + expression=4, expression_type="PYTHON_9.3") + + arcpy.Sort_management("optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved_commodity", + [["SORT_FIELD", "ASCENDING"]]) + + # Delete temp fc's + arcpy.Delete_management("optimized_route_segments_dissolved_tmp") + arcpy.Delete_management("optimized_route_segments_split_tmp") + arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") + arcpy.Delete_management("optimized_route_segments_lyr") + arcpy.Delete_management("dissolved_segments_lyr") + + # Copy to permanent fc (unique to commodity name) + arcpy.CopyFeatures_management("optimized_route_segments_dissolved_commodity", + "optimized_route_segments_dissolved_" + layer_name) diff --git a/program/ftot_networkx.py b/program/ftot_networkx.py index ea68c8b..0a5ab75 100644 --- a/program/ftot_networkx.py +++ b/program/ftot_networkx.py @@ -1,1256 +1,1256 @@ -# ------------------------------------------------------------------------------ -# Name: ftot_networkx.py -# -# Purpose: the purpose of this module is to handle all the NetworkX methods and operations -# necessary to go between FTOT layers: GIS, sqlite DB, etc. -# Revised: 6/15/21 -# ------------------------------------------------------------------------------ - -import networkx as nx -import sqlite3 -from shutil import rmtree -import datetime -import ftot_supporting -import arcpy -import os -import multiprocessing -import math -from ftot_pulp import commodity_mode_setup - - -# ----------------------------------------------------------------------------- - - -def graph(the_scenario, logger): - - # check for permitted modes before creating nX graph - check_permitted_modes(the_scenario, logger) - - # export the assets from GIS export_fcs_from_main_gdb - export_fcs_from_main_gdb(the_scenario, logger) - - # create the networkx multidigraph - G = make_networkx_graph(the_scenario, logger) - - # clean up the networkx graph to preserve connectivity - clean_networkx_graph(the_scenario, G, logger) - - # cache the digraph to the db and store the route_cost_scaling factor - digraph_to_db(the_scenario, G, logger) - - # cost the network in the db - set_network_costs_in_db(the_scenario, logger) - - # generate shortest paths through the network - presolve_network(the_scenario, G, logger) - - # eliminate the graph and related shape files before moving on - delete_shape_files(the_scenario, logger) - G = nx.null_graph() - - -# ----------------------------------------------------------------------------- - - -def delete_shape_files(the_scenario, logger): - # delete temporary files - logger.debug("start: delete the temp_networkx_shp_files dir") - input_path = the_scenario.networkx_files_dir - rmtree(input_path) - logger.debug("finish: delete the temp_networkx_shp_files dir") - - -# ----------------------------------------------------------------------------- - - -# Scan the XML and input_data to ensure that pipelines are permitted and relevant -def check_permitted_modes(the_scenario, logger): - logger.debug("start: check permitted modes") - commodity_mode_dict = commodity_mode_setup(the_scenario, logger) - with sqlite3.connect(the_scenario.main_db) as db_cur: - # get pipeline records with an allow_yn == y - sql = "select * from commodity_mode where mode like 'pipeline%' and allowed_yn like 'y';" - pipeline_allowed = db_cur.execute(sql).fetchall() - if not pipeline_allowed: - logger.info("pipelines are not allowed") - new_permitted_mode_list = [] - for mode in the_scenario.permittedModes: - if 'pipeline' not in mode: - new_permitted_mode_list.append(mode) - elif 'pipeline' in mode: - continue # we don't want to include pipeline fcs if no csv exists to specify product flow - the_scenario.permittedModes = new_permitted_mode_list - logger.debug("finish: check permitted modes") - - -# ----------------------------------------------------------------------------- - - -def make_networkx_graph(the_scenario, logger): - # High level work flow: - # ------------------------ - # make_networkx_graph - # create the multidigraph - # convert the node labels to integers - # reverse the graph and compose with self - - logger.info("start: make_networkx_graph") - start_time = datetime.datetime.now() - - # read the shapefiles in the customized read_shp method - input_path = the_scenario.networkx_files_dir - - logger.debug("start: read_shp") - G = read_shp(input_path, logger, simplify=True, - geom_attrs=False, strict=True) # note this is custom and not nx.read_shp() - - # cleanup the node labels - logger.debug("start: convert node labels") - G = nx.convert_node_labels_to_integers(G, first_label=0, ordering='default', label_attribute="x_y_location") - - # create a reversed graph - logger.debug("start: reverse G graph to H") - H = G.reverse() # this is a reversed version of the graph. - - # set a new attribute for every edge that says its a "reversed" link - # we will use this to delete edges that shouldn't be reversed later. - logger.debug("start: set 'reversed' attribute in H") - nx.set_edge_attributes(H, 1, "REVERSED") - - # add the two graphs together - logger.debug("start: compose G and H") - G = nx.compose(G, H) - - # print out some stats on the Graph - logger.info("Number of nodes in the raw graph: {}".format(G.order())) - logger.info("Number of edges in the raw graph: {}".format(G.size())) - - logger.debug( - "finished: make_networkx_graph: Runtime (HMS): \t{}".format( - ftot_supporting.get_total_runtime_string(start_time))) - - return G - - -# ----------------------------------------------------------------------------- - - -def presolve_network(the_scenario, G, logger): - logger.debug("start: presolve_network") - - # Check NDR conditions before calculating shortest paths - update_ndr_parameter(the_scenario, logger) - - # Create a table to hold the shortest edges - with sqlite3.connect(the_scenario.main_db) as db_cur: - # clean up the db - sql = "drop table if exists shortest_edges;" - db_cur.execute(sql) - - sql = """ - create table if not exists shortest_edges (from_node_id INT, to_node_id INT, edge_id INT, - CONSTRAINT unique_from_to_edge_id_tuple UNIQUE (from_node_id, to_node_id, edge_id)); - """ - db_cur.execute(sql) - - # Create a table to hold routes - with sqlite3.connect(the_scenario.main_db) as db_cur: - # clean up the db - sql = "drop table if exists route_edges;" - db_cur.execute(sql) - - sql = """ - create table if not exists route_edges (from_node_id INT, to_node_id INT, edge_id INT, - scenario_rt_id INT, rt_order_ind INT); - """ - db_cur.execute(sql) - - # If NDR_On = False (default case), then skip calculating shortest paths - if not the_scenario.ndrOn: - with sqlite3.connect(the_scenario.main_db) as db_cur: - sql = """ - insert or ignore into shortest_edges - select from_node_id, to_node_id, edge_id - from networkx_edges; - """ - logger.debug("NDR skipped and all edges added to shortest_edges table") - db_cur.execute(sql) - db_cur.commit() - logger.debug("finish: presolve_network") - return - - # Get phases_of_matter in the scenario - phases_of_matter_in_scenario = get_phases_of_matter_in_scenario(the_scenario, logger) - - # Determine the weights for each phase of matter in scenario associated with the edges in the nX graph - nx_graph_weighting(the_scenario, G, phases_of_matter_in_scenario, logger) - - # Make subgraphs for combination of permitted modes - commodity_subgraph_dict = make_mode_subgraphs(the_scenario, G, logger) - - # Create a dictionary of edge_ids from the database which is used later to uniquely identify edges - edge_id_dict = find_edge_ids(the_scenario, logger) - - # Make origin-destination pairs where od_pairs is a dictionary keyed off [commodity_id, target, source], - # value is a list of [scenario_rt_id, phase_of_matter] for that key - od_pairs = make_od_pairs(the_scenario, logger) - - # Use multi-processing to determine shortest_paths for each target in the od_pairs dictionary - manager = multiprocessing.Manager() - all_route_edges = manager.list() - no_path_pairs = manager.list() - logger.debug("multiprocessing.cpu_count() = {}".format(multiprocessing.cpu_count())) - - # To parallelize computations, assign a set of targets to be passed to each processor - stuff_to_pass = [] - logger.debug("start: identify shortest_path between each o-d pair by commodity") - # by commodity and destination - for commodity_id in od_pairs: - phase_of_matter = od_pairs[commodity_id]['phase_of_matter'] - allowed_modes = commodity_subgraph_dict[commodity_id]['modes'] - for a_target in od_pairs[commodity_id]['targets'].keys(): - stuff_to_pass.append([commodity_subgraph_dict[commodity_id]['subgraph'], - od_pairs[commodity_id]['targets'][a_target], - a_target, all_route_edges, no_path_pairs, - edge_id_dict, phase_of_matter, allowed_modes]) - - # Allow multiprocessing, with no more than 75% of cores to be used, rounding down if necessary - logger.info("start: the multiprocessing route solve.") - processors_to_save = int(math.ceil(multiprocessing.cpu_count() * 0.25)) - processors_to_use = multiprocessing.cpu_count() - processors_to_save - logger.info("number of CPUs to use = {}".format(processors_to_use)) - - pool = multiprocessing.Pool(processes=processors_to_use) - try: - pool.map(multi_shortest_paths, stuff_to_pass) - except Exception as e: - pool.close() - pool.terminate() - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - pool.close() - pool.join() - - logger.info("end: identify shortest_path between each o-d pair") - - # Log any origin-destination pairs without a shortest path - if no_path_pairs: - logger.warning("Cannot identify shortest paths for {} o-d pairs; see log file list".format(len(no_path_pairs))) - for i in no_path_pairs: - s, t, rt_id = i - logger.debug("Missing shortest path for source {}, target {}, scenario_route_id {}".format(s, t, rt_id)) - - with sqlite3.connect(the_scenario.main_db) as db_cur: - sql = """ - insert into route_edges - (from_node_id, to_node_id, edge_id, scenario_rt_id, rt_order_ind) - values (?,?,?,?,?); - """ - - logger.debug("start: update route_edges table") - db_cur.executemany(sql, all_route_edges) - db_cur.commit() - logger.debug("end: update route_edges table") - - with sqlite3.connect(the_scenario.main_db) as db_cur: - sql = """ - insert or ignore into shortest_edges - (from_node_id, to_node_id, edge_id) - select distinct from_node_id, to_node_id, edge_id - from route_edges; - """ - logger.debug("start: update shortest_edges table") - db_cur.execute(sql) - db_cur.commit() - logger.debug("end: update shortest_edges table") - - logger.debug("finish: presolve_network") - - -# ----------------------------------------------------------------------------- - - -# Ensure shortest paths not calculated in presence of candidate generation, capacity, max transport distance -def update_ndr_parameter(the_scenario, logger): - logger.debug("start: check NDR conditions") - new_ndr_parameter = the_scenario.ndrOn - - # if capacity is enabled, then skip NDR - if the_scenario.capacityOn: - new_ndr_parameter = False - logger.debug("NDR de-activated due to capacity enforcement") - - # if candidate generation is active, then skip NDR - if the_scenario.processors_candidate_slate_data != 'None': - new_ndr_parameter = False - logger.debug("NDR de-activated due to candidate generation") - - # if max_transport_distance field is used in commodities table, then skip NDR - with sqlite3.connect(the_scenario.main_db) as main_db_con: - # get count of commodities with a specified max_transport_distance - sql = "select count(commodity_id) from commodities where max_transport_distance is not null;" - db_cur = main_db_con.execute(sql) - count = db_cur.fetchone()[0] - if count > 0: - new_ndr_parameter = False - logger.debug("NDR de-activated due to use of max transport distance") - - the_scenario.ndrOn = new_ndr_parameter - logger.debug("finish: check NDR conditions") - - -# ----------------------------------------------------------------------------- - - -# This method uses a shortest_path algorithm from the nx library to flag edges in the -# network that are a part of the shortest path connecting an origin to a destination -# for each commodity -def multi_shortest_paths(stuff_to_pass): - global all_route_edges, no_path_pairs - G, sources, target, all_route_edges, no_path_pairs, edge_id_dict, phase_of_matter, allowed_modes = stuff_to_pass - t = target - - shortest_paths_to_t = nx.shortest_path(G, target=t, weight='{}_weight'.format(phase_of_matter)) - for a_source in sources: - s = a_source - # This accounts for when a_source may not be connected to t, - # as is the case when certain modes may not be permitted - if a_source not in shortest_paths_to_t: - for i in sources[a_source]: - rt_id = i - no_path_pairs.append((s, t, rt_id)) - continue - for i in sources[a_source]: - rt_id = i - for index, from_node in enumerate(shortest_paths_to_t[s]): - if index < (len(shortest_paths_to_t[s]) - 1): - to_node = shortest_paths_to_t[s][index + 1] - # find the correct edge_id on the shortest path - min_route_cost = 999999999 - min_edge_id = None - for j in edge_id_dict[to_node][from_node]: - edge_id, mode_source, route_cost = j - if mode_source in allowed_modes: - if route_cost < min_route_cost: - min_route_cost = route_cost - min_edge_id = edge_id - if min_edge_id is None: - error = """something went wrong finding the edge_id from node {} to node {} - for scenario_rt_id {} in shortest path algorithm""".format(from_node, to_node, rt_id) - raise Exception(error) - all_route_edges.append((from_node, to_node, min_edge_id, rt_id, index + 1)) - - -# ----------------------------------------------------------------------------- - - -# Assigns for each link in the networkX graph a weight for each phase of matter -# which mirrors the cost of each edge for the optimizer -def nx_graph_weighting(the_scenario, G, phases_of_matter_in_scenario, logger): - - # pull the route cost for all edges in the graph - logger.debug("start: assign edge weights to networkX graph") - for phase_of_matter in phases_of_matter_in_scenario: - # initialize the edge weight variable to something large to be overwritten in the loop - nx.set_edge_attributes(G, 999999999, name='{}_weight'.format(phase_of_matter)) - - for (u, v, c, d) in G.edges(keys=True, data='route_cost_scaling', default=False): - from_node_id = u - to_node_id = v - route_cost_scaling = G.edges[(u, v, c)]['route_cost_scaling'] - mileage = G.edges[(u, v, c)]['MILES'] - source = G.edges[(u, v, c)]['source'] - artificial = G.edges[(u, v, c)]['Artificial'] - - # calculate route_cost for all edges and phases of matter to mirror network costs in db - for phase_of_matter in phases_of_matter_in_scenario: - weight = get_network_link_cost(the_scenario, phase_of_matter, source, artificial, logger) - if 'pipeline' not in source: - if artificial == 0: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = mileage * route_cost_scaling * weight - elif artificial == 1: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight - elif artificial == 2: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight / 2.00 - else: - logger.warning("artificial code of {} is not supported!".format(artificial)) - else: - # inflate the cost of pipelines for 'solid' to avoid forming a shortest path with them - if phase_of_matter == 'solid': - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = 1000000 - else: - if artificial == 0: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = route_cost_scaling - elif artificial == 1: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = mileage * weight - elif artificial == 2: - G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight / 2.00 - else: - logger.warning("artificial code of {} is not supported!".format(artificial)) - - logger.debug("end: assign edge weights to networkX graph") - - -# ----------------------------------------------------------------------------- - - -# Returns a dictionary of NetworkX graphs keyed off commodity_id with 'modes' and 'subgraph' keys -def make_mode_subgraphs(the_scenario, G, logger): - logger.debug("start: create mode subgraph dictionary") - - logger.debug("start: pull commodity mode from SQL") - with sqlite3.connect(the_scenario.main_db) as db_cur: - sql = "select mode, commodity_id from commodity_mode where allowed_yn like 'y';" - commodity_mode_data = db_cur.execute(sql).fetchall() - commodity_subgraph_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - if commodity_id not in commodity_subgraph_dict: - commodity_subgraph_dict[commodity_id] = {} - commodity_subgraph_dict[commodity_id]['modes'] = [] - commodity_subgraph_dict[commodity_id]['modes'].append(mode) - logger.debug("end: pull commodity mode from SQL") - - for k in commodity_subgraph_dict: - commodity_subgraph_dict[k]['modes'] = sorted(commodity_subgraph_dict[k]['modes']) - - # check if commodity mode input file exists - if not os.path.exists(the_scenario.commodity_mode_data): - # all commodities are allowed on all permitted modes and use graph G - logger.debug("no commodity_mode_data file: {}".format(the_scenario.commodity_mode_data)) - logger.debug("all commodities allowed on all permitted modes") - for k in commodity_subgraph_dict: - commodity_subgraph_dict[k]['subgraph'] = G - else: - # generate subgraphs for each unique combination of allowed modes - logger.debug("creating subgraphs for each unique set of allowed modes") - # create a dictionary of subgraphs indexed by 'modes' - subgraph_dict = {} - for k in commodity_subgraph_dict: - allowed_modes = str(commodity_subgraph_dict[k]['modes']) - if allowed_modes not in subgraph_dict: - subgraph_dict[allowed_modes] = nx.DiGraph(((source, target, attr) for source, target, attr in - G.edges(data=True) if attr['MODE_TYPE'] in - commodity_subgraph_dict[k]['modes'])) - - # assign subgraphs to commodities - for k in commodity_subgraph_dict: - commodity_subgraph_dict[k]['subgraph'] = subgraph_dict[str(commodity_subgraph_dict[k]['modes'])] - - logger.debug("end: create mode subgraph dictionary") - - return commodity_subgraph_dict - - -# ----------------------------------------------------------------------------- - - -# Returns a dictionary of edge_ids keyed off (to_node_id, from_node_id) -def find_edge_ids(the_scenario, logger): - logger.debug("start: create edge_id dictionary") - logger.debug("start: pull edge_ids from SQL") - with sqlite3.connect(the_scenario.main_db) as db_cur: - sql = """ - select ne.from_node_id, ne.to_node_id, ne.edge_id, ne.mode_source, nec.route_cost - from networkx_edges ne - left join networkx_edge_costs nec - on ne.edge_id = nec.edge_id; - """ - sql_list = db_cur.execute(sql).fetchall() - logger.debug("end: pull edge_ids from SQL") - - edge_id_dict = {} - - for row in sql_list: - from_node = row[0] - to_node = row[1] - edge_id = row[2] - mode_source = row[3] - route_cost = row[4] - if to_node not in edge_id_dict: - edge_id_dict[to_node] = {} - if from_node not in edge_id_dict[to_node].keys(): - edge_id_dict[to_node][from_node] = [] - edge_id_dict[to_node][from_node].append([edge_id, mode_source, route_cost]) - - logger.debug("end: create edge_id dictionary") - - return edge_id_dict - - -# ----------------------------------------------------------------------------- - - -# Creates a dictionary of all feasible origin-destination pairs, including: -# RMP-DEST, RMP-PROC, PROC-DEST, etc., indexed by [commodity_id][target][source] -def make_od_pairs(the_scenario, logger): - with sqlite3.connect(the_scenario.main_db) as db_cur: - # Create a table for od_pairs in the database - logger.info("start: create o-d pairs table") - sql = "drop table if exists od_pairs;" - db_cur.execute(sql) - - sql = ''' - create table od_pairs(scenario_rt_id INTEGER PRIMARY KEY, from_location_id integer, to_location_id integer, - from_facility_id integer, to_facility_id integer, commodity_id integer, phase_of_matter text, route_status text, - from_node_id INTEGER, to_node_id INTEGER, from_location_1 INTEGER, to_location_1 INTEGER); - ''' - db_cur.execute(sql) - - # Populate the od_pairs table from a temporary table that collects both origins and destinations - sql = "drop table if exists tmp_connected_facilities_with_commodities;" - db_cur.execute(sql) - - sql = ''' - create table tmp_connected_facilities_with_commodities as - select - facilities.facility_id, - facilities.location_id, - facilities.facility_name, - facility_type_id.facility_type, - ignore_facility, - facility_commodities.commodity_id, - facility_commodities.io, - commodities.phase_of_matter, - networkx_nodes.node_id, - networkx_nodes.location_1 - from facilities - join facility_commodities on - facility_commodities.facility_id = facilities.facility_ID - join commodities on - facility_commodities.commodity_id = commodities.commodity_id - join facility_type_id on - facility_type_id.facility_type_id = facilities.facility_type_id - join networkx_nodes on - networkx_nodes.location_id = facilities.location_id - where ignore_facility = 'false'; - ''' - db_cur.execute(sql) - - sql = ''' - insert into od_pairs (from_location_id, to_location_id, from_facility_id, to_facility_id, commodity_id, - phase_of_matter, from_node_id, to_node_id, from_location_1, to_location_1) - select distinct - origin.location_id AS from_location_id, - destination.location_id AS to_location_id, - origin.facility_id AS from_facility_id, - destination.facility_id AS to_facility_id, - origin.commodity_id AS commodity_id, - origin.phase_of_matter AS phase_of_matter, - origin.node_id AS from_node_id, - destination.node_id AS to_node_id, - origin.location_1 AS from_location_1, - destination.location_1 AS to_location_1 - from - tmp_connected_facilities_with_commodities as origin - inner join - tmp_connected_facilities_with_commodities as destination ON - CASE - WHEN origin.facility_type <> 'processor' or destination.facility_type <> 'processor' -- THE NORMAL CASE, RMP->PROC, RMP->DEST, or PROC->DEST - THEN - origin.facility_type <> destination.facility_type -- not the same facility_type - and - origin.commodity_id = destination.commodity_id -- match on commodity - and - origin.facility_id <> destination.facility_id -- not the same facility - and - origin.facility_type <> "ultimate_destination" -- restrict origin types - and - destination.facility_type <> "raw_material_producer" -- restrict destination types - and - destination.facility_type <> "raw_material_producer_as_processor" -- restrict other destination types - and - origin.location_1 like '%_OUT' -- restrict to the correct out/in node_id's - AND destination.location_1 like '%_IN' - ELSE -- THE CASE WHEN PROCESSORS ARE SENDING STUFF TO OTHER PROCESSORS - origin.io = 'o' -- make sure processors origins send outputs - and - destination.io = 'i' -- make sure processors destinations receive inputs - and - origin.commodity_id = destination.commodity_id -- match on commodity - and - origin.facility_id <> destination.facility_id -- not the same facility - and - origin.facility_type <> "ultimate_destination" -- restrict origin types - and - destination.facility_type <> "raw_material_producer" -- restrict destination types - and - destination.facility_type <> "raw_material_producer_as_processor" -- restrict other destination types - and - origin.location_1 like '%_OUT' -- restrict to the correct out/in node_id's - AND destination.location_1 like '%_IN' - END; - ''' - db_cur.execute(sql) - - logger.debug("drop the tmp_connected_facilities_with_commodities table") - db_cur.execute("drop table if exists tmp_connected_facilities_with_commodities;") - - logger.info("end: create o-d pairs table") - - # Fetch all od-pairs, ordered by target - sql = ''' - SELECT to_node_id, from_node_id, scenario_rt_id, commodity_id, phase_of_matter - FROM od_pairs ORDER BY to_node_id DESC; - ''' - sql_list = db_cur.execute(sql).fetchall() - - # Loop through the od_pairs - od_pairs = {} - for row in sql_list: - target = row[0] - source = row[1] - scenario_rt_id = row[2] - commodity_id = row[3] - phase_of_matter = row[4] - if commodity_id not in od_pairs: - od_pairs[commodity_id] = {} - od_pairs[commodity_id]['phase_of_matter'] = phase_of_matter - od_pairs[commodity_id]['targets'] = {} - if target not in od_pairs[commodity_id]['targets'].keys(): - od_pairs[commodity_id]['targets'][target] = {} - if source not in od_pairs[commodity_id]['targets'][target].keys(): - od_pairs[commodity_id]['targets'][target][source] = [] - od_pairs[commodity_id]['targets'][target][source].append(scenario_rt_id) - - return od_pairs - - -# ----------------------------------------------------------------------------- - - -def export_fcs_from_main_gdb(the_scenario, logger): - # export fcs from the main.GDB to individual shapefiles - logger.info("start: export_fcs_from_main_gdb") - start_time = datetime.datetime.now() - - # export network and locations fc's to shapefiles - main_gdb = the_scenario.main_gdb - output_path = the_scenario.networkx_files_dir - input_features = [] - - logger.debug("start: create temp_networkx_shp_files dir") - if os.path.exists(output_path): - logger.debug("deleting pre-existing temp_networkx_shp_files dir") - rmtree(output_path) - - if not os.path.exists(output_path): - logger.debug("creating new temp_networkx_shp_files dir") - os.makedirs(output_path) - - location_list = ['\\locations', '\\network\\intermodal', '\\network\\locks'] - - # only add shape files associated with modes that are permitted in the scenario file - for mode in the_scenario.permittedModes: - location_list.append('\\network\\{}'.format(mode)) - - # get the locations and network feature layers - for fc in location_list: - input_features.append(main_gdb + fc) - arcpy.FeatureClassToShapefile_conversion(Input_Features=input_features, Output_Folder=output_path) - - logger.debug("finished: export_fcs_from_main_gdb: Runtime (HMS): \t{}".format( - ftot_supporting.get_total_runtime_string(start_time))) - - -# ------------------------------------------------------------------------------ - - -def clean_networkx_graph(the_scenario, G, logger): - # ------------------------------------------------------------------------- - # renamed clean_networkx_graph () - # remove reversed links for pipeline - # selectively remove links for location _IN and _OUT nodes - # preserve the route_cost_scaling factor in an attribute by phase of matter - - logger.info("start: clean_networkx_graph") - start_time = datetime.datetime.now() - - logger.debug("Processing the {} edges in the uncosted graph.".format(G.size())) - - # use the artificial and reversed attribute to determine if - # the link is kept - # ------------------------------------------------------------- - edge_attrs = {} # for storing the edge attributes which are set all at once - deleted_edge_count = 0 - - for u, v, keys, artificial in list(G.edges(data='Artificial', keys=True)): - - # initialize the route_cost_scaling variable to something - # absurd so we know if its getting set properly in the loop: - route_cost_scaling = -999999999 - - # check if the link is reversed - if 'REVERSED' in G.edges[u, v, keys]: - reversed_link = G.edges[u, v, keys]['REVERSED'] - else: - reversed_link = 0 - - # check if capacity is 0 - not currently considered here - # Network Edges - artificial == 0 - # ----------------------------------- - if artificial == 0: - - # check the mode type - # ---------------------- - mode_type = G.edges[u, v, keys]['MODE_TYPE'] - - # set the mode specific weights - # ----------------------------- - - if mode_type == "rail": - d_code = G.edges[u, v, keys]["DENSITY_CO"] - if d_code in [7]: - route_cost_scaling = the_scenario.rail_dc_7 - elif d_code in [6]: - route_cost_scaling = the_scenario.rail_dc_6 - elif d_code in [5]: - route_cost_scaling = the_scenario.rail_dc_5 - elif d_code in [4]: - route_cost_scaling = the_scenario.rail_dc_4 - elif d_code in [3]: - route_cost_scaling = the_scenario.rail_dc_3 - elif d_code in [2]: - route_cost_scaling = the_scenario.rail_dc_2 - elif d_code in [1]: - route_cost_scaling = the_scenario.rail_dc_1 - elif d_code in [0]: - route_cost_scaling = the_scenario.rail_dc_0 - else: - logger.warning("The d_code {} is not supported".format(d_code)) - - elif mode_type == "water": - - # get the total vol of water traffic - tot_vol = G.edges[u, v, keys]['TOT_UP_DWN'] - if tot_vol >= 10000000: - route_cost_scaling = the_scenario.water_high_vol - elif 1000000 <= tot_vol < 10000000: - route_cost_scaling = the_scenario.water_med_vol - elif 1 <= tot_vol < 1000000: - route_cost_scaling = the_scenario.water_low_vol - else: - route_cost_scaling = the_scenario.water_no_vol - - elif mode_type == "road": - - # get fclass - fclass = G.edges[u, v, keys]['FCLASS'] - if fclass in [1]: - route_cost_scaling = the_scenario.truck_interstate - elif fclass in [2, 3]: - route_cost_scaling = the_scenario.truck_pr_art - elif fclass in [4]: - route_cost_scaling = the_scenario.truck_m_art - else: - route_cost_scaling = the_scenario.truck_local - - elif 'pipeline' in mode_type: - if reversed_link == 1: - G.remove_edge(u, v, keys) - deleted_edge_count += 1 - continue # move on to the next edge - else: - route_cost_scaling = (((float(G.edges[u, v, keys]['base_rate']) / 100) / 42.0) * 1000.0) - - # Intermodal Edges - artificial == 2 - # ------------------------------------ - elif artificial == 2: - # set it to 1 because we'll multiply by the appropriate - # link_cost later for transloading - route_cost_scaling = 1 - - # Artificial Edge - artificial == 1 - # ---------------------------------- - # need to check if its an IN location or an OUT location and delete selectively. - # assume always connecting from the node to the network. - # so _OUT locations should delete the reversed link - # _IN locations should delete the non-reversed link. - elif artificial == 1: - # delete edges we dont want - - try: - if G.edges[u, v, keys]['LOCATION_1'].find("_OUT") > -1 and reversed_link == 1: - G.remove_edge(u, v, keys) - deleted_edge_count += 1 - continue # move on to the next edge - elif G.edges[u, v, keys]['LOCATION_1'].find("_IN") > -1 and reversed_link == 0: - G.remove_edge(u, v, keys) - deleted_edge_count += 1 - continue # move on to the next edge - - # there is no scaling of artificial links. - # the cost_penalty is calculated in get_network_link_cost() - else: - route_cost_scaling = 1 - except: - logger.warning("the following keys didn't work:u - {}, v- {}".format(u, v)) - else: - logger.warning("found an edge without artificial attribute: {} ") - continue - - edge_attrs[u, v, keys] = { - 'route_cost_scaling': route_cost_scaling - } - - nx.set_edge_attributes(G, edge_attrs) - - # print out some stats on the Graph - logger.info("Number of nodes in the clean graph: {}".format(G.order())) - logger.info("Number of edges in the clean graph: {}".format(G.size())) - - logger.debug("finished: clean_networkx_graph: Runtime (HMS): \t{}".format( - ftot_supporting.get_total_runtime_string(start_time))) - - return G - - -# ------------------------------------------------------------------------------ - - -def get_network_link_cost(the_scenario, phase_of_matter, mode, artificial, logger): - # three types of artificial links: - # (0 = network edge, 2 = intermodal, 1 = artificial link btw facility location and network edge) - # add the appropriate cost to the network edges based on phase of matter - - if phase_of_matter == "solid": - # set the mode costs - truck_base_cost = the_scenario.solid_truck_base_cost.magnitude - railroad_class_1_cost = the_scenario.solid_railroad_class_1_cost.magnitude - barge_cost = the_scenario.solid_barge_cost.magnitude - transloading_cost = the_scenario.solid_transloading_cost.magnitude - rail_short_haul_penalty = the_scenario.solid_rail_short_haul_penalty.magnitude - water_short_haul_penalty = the_scenario.solid_water_short_haul_penalty.magnitude - - elif phase_of_matter == "liquid": - # set the mode costs - truck_base_cost = the_scenario.liquid_truck_base_cost.magnitude - railroad_class_1_cost = the_scenario.liquid_railroad_class_1_cost.magnitude - barge_cost = the_scenario.liquid_barge_cost.magnitude - transloading_cost = the_scenario.liquid_transloading_cost.magnitude - rail_short_haul_penalty = the_scenario.liquid_rail_short_haul_penalty.magnitude - water_short_haul_penalty = the_scenario.liquid_water_short_haul_penalty.magnitude - - else: - logger.error("the phase of matter: -- {} -- is not supported. returning") - raise NotImplementedError - - if artificial == 1: - # add a fixed cost penalty to the routing cost for artificial links - if mode == "rail": - link_cost = rail_short_haul_penalty / 2.0 - # Divide by 2 is to ensure the penalty is not doubled-- it is applied on artificial links on both ends - elif mode == "water": - link_cost = water_short_haul_penalty / 2.0 - # Divide by 2 is to ensure the penalty is not doubled-- it is applied on artificial links on both ends - elif 'pipeline' in mode: - # this cost penalty was calculated by looking at the average per mile base rate. - link_cost = 0.19 - # no mileage multiplier here for pipeline as unlike rail/water, we do not want to disproportionally - # discourage short movements - # Multiplier will be applied based on actual link mileage when scenario costs are actually set - else: - link_cost = the_scenario.truck_local * truck_base_cost # for road - - elif artificial == 2: - # phase of mater is determined above - link_cost = transloading_cost - - elif artificial == 0: - if mode == "road": - link_cost = truck_base_cost - elif mode == "rail": - link_cost = railroad_class_1_cost - elif mode == "water": - link_cost = barge_cost - elif mode == "pipeline_crude_trf_rts": - link_cost = 1 # so we multiply by the base_rate - elif mode == "pipeline_prod_trf_rts": - link_cost = 1 # so we multiply by base_rate - - return link_cost - - -# ---------------------------------------------------------------------------- - - -def get_phases_of_matter_in_scenario(the_scenario, logger): - logger.debug("start: get_phases_of_matter_in_scenario()") - - phases_of_matter_in_scenario = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - sql = "select count(distinct phase_of_matter) from commodities where phase_of_matter is not null;" - db_cur = main_db_con.execute(sql) - - count = db_cur.fetchone()[0] - logger.debug("phases_of_matter in the scenario: {}".format(count)) - - if not count: - error = "No phases of matter in the scenario: {}...returning".format(count) - logger.error(error) - raise Exception(error) - - elif count: - sql = "select phase_of_matter from commodities where phase_of_matter is not null group by phase_of_matter" - db_cur = main_db_con.execute(sql) - for row in db_cur: - phases_of_matter_in_scenario.append(row[0]) - else: - logger.warning("Something went wrong in get_phases_of_matter_in_scenario()") - error = "Count phases of matter to route: {}".format(str(count)) - logger.error(error) - raise Exception(error) - - logger.debug("end: get_phases_of_matter_in_scenario()") - return phases_of_matter_in_scenario - - -# ----------------------------------------------------------------------------- - - -# set the network costs in the db by phase_of_matter -def set_network_costs_in_db(the_scenario, logger): - - logger.info("start: set_network_costs_in_db") - with sqlite3.connect(the_scenario.main_db) as db_con: - # clean up the db - sql = "drop table if exists networkx_edge_costs" - db_con.execute(sql) - - sql = "create table if not exists networkx_edge_costs " \ - "(edge_id INTEGER, phase_of_matter_id INT, route_cost REAL, dollar_cost REAL)" - db_con.execute(sql) - - # build up the network edges cost by phase of matter - edge_cost_list = [] - - # get phases_of_matter in the scenario - phases_of_matter_in_scenario = get_phases_of_matter_in_scenario(the_scenario, logger) - - # loop through each edge in the networkx_edges table - sql = "select edge_id, mode_source, artificial, miles, route_cost_scaling from networkx_edges" - db_cur = db_con.execute(sql) - for row in db_cur: - - edge_id = row[0] - mode_source = row[1] - artificial = row[2] - miles = row[3] - route_cost_scaling = row[4] - - for phase_of_matter in phases_of_matter_in_scenario: - - # skip pipeline and solid phase of matter - if phase_of_matter == 'solid' and 'pipeline' in mode_source: - continue - - # otherwise, go ahead and get the link cost - link_cost = get_network_link_cost(the_scenario, phase_of_matter, mode_source, artificial, logger) - - if artificial == 0: - # road, rail, and water - if 'pipeline' not in mode_source: - dollar_cost = miles * link_cost # link_cost is taken from the scenario file - route_cost = dollar_cost * route_cost_scaling # this includes impedance - - else: - # if artificial = 0, route_cost_scaling = base rate - # we use the route_cost_scaling for this since its set in the GIS - # not in the scenario xml file. - - dollar_cost = route_cost_scaling # this is the base rate - route_cost = route_cost_scaling # this is the base rate - - elif artificial == 1: - # we don't want to add the cost penalty to the dollar cost for artificial links - dollar_cost = 0 - - if 'pipeline' not in mode_source: - # the routing_cost should have the artificial link penalty - # and artificial link penalties shouldn't be scaled by mileage. - route_cost = link_cost - - else: - # For pipeline we don't want to penalize short movements disproportionately, - # so scale penalty by miles. This gives art links for pipeline a modest routing cost - route_cost = link_cost * miles - - elif artificial == 2: - dollar_cost = link_cost / 2.00 # this is the transloading fee - # For now dividing by 2 to ensure that the transloading fee is not applied twice - # (e.g. on way in and on way out) - route_cost = link_cost / 2.00 # same as above. - - else: - logger.warning("artificial code of {} is not supported!".format(artificial)) - - edge_cost_list.append([edge_id, phase_of_matter, route_cost, dollar_cost]) - - if edge_cost_list: - update_sql = """ - INSERT into networkx_edge_costs - values (?,?,?,?) - ;""" - - db_con.executemany(update_sql, edge_cost_list) - logger.debug("start: networkx_edge_costs commit") - db_con.commit() - logger.debug("finish: networkx_edge_costs commit") - - logger.debug("finished: set_network_costs_in_db") - - -# ----------------------------------------------------------------------------- - - -def digraph_to_db(the_scenario, G, logger): - # moves the networkX digraph into the database for the pulp handshake - - logger.info("start: digraph_to_db") - with sqlite3.connect(the_scenario.main_db) as db_con: - - # clean up the db - sql = "drop table if exists networkx_nodes" - db_con.execute(sql) - - sql = "create table if not exists networkx_nodes (node_id INT, source TEXT, source_OID integer, location_1 " \ - "TEXT, location_id TEXT, shape_x REAL, shape_y REAL)" - db_con.execute(sql) - - # loop through the nodes in the digraph and set them in the db - # nodes will be either locations (with a location_id), or nodes connecting - # network edges (with no location info). - node_list = [] - - for node in G.nodes(): - source = None - source_oid = None - location_1 = None - location_id = None - shape_x = None - shape_y = None - - if 'source' in G.nodes[node]: - source = G.nodes[node]['source'] - - if 'source_OID' in G.nodes[node]: - source_oid = G.nodes[node]['source_OID'] - - if 'location_1' in G.nodes[node]: # for locations - location_1 = G.nodes[node]['location_1'] - location_id = G.nodes[node]['location_i'] - - if 'x_y_location' in G.nodes[node]: - shape_x = G.nodes[node]['x_y_location'][0] - shape_y = G.nodes[node]['x_y_location'][1] - - node_list.append([node, source, source_oid, location_1, location_id, shape_x, shape_y]) - - if node_list: - update_sql = """ - INSERT into networkx_nodes - values (?,?,?,?,?,?,?) - ;""" - - db_con.executemany(update_sql, node_list) - db_con.commit() - logger.debug("finished network_x nodes commit") - - # loop through the edges in the digraph and insert them into the db. - # ------------------------------------------------------------------- - edge_list = [] - with sqlite3.connect(the_scenario.main_db) as db_con: - - # clean up the db - sql = "drop table if exists networkx_edges" - db_con.execute(sql) - - sql = "create table if not exists networkx_edges (edge_id INTEGER PRIMARY KEY, from_node_id INT, to_node_id " \ - "INT, artificial INT, mode_source TEXT, mode_source_oid INT, miles REAL, route_cost_scaling REAL, " \ - "capacity INT, volume REAL, VCR REAL)" - db_con.execute(sql) - - for (u, v, c, d) in G.edges(keys=True, data='route_cost_scaling', default=False): - from_node_id = u - to_node_id = v - miles = G.edges[(u, v, c)]['MILES'] - artificial = G.edges[(u, v, c)]['Artificial'] - mode_source = G.edges[(u, v, c)]['MODE_TYPE'] - mode_source_oid = G.edges[(u, v, c)]['source_OID'] - - if mode_source in ['rail', 'road']: - volume = G.edges[(u, v, c)]['Volume'] - vcr = G.edges[(u, v, c)]['VCR'] - capacity = G.edges[(u, v, c)]['Capacity'] - else: - volume = None - vcr = None - capacity = None - - if capacity == 0: - capacity = None - logger.detailed_debug("link capacity == 0, setting to None".format(G.edges[(u, v, c)])) - - if 'route_cost_scaling' in G.edges[(u, v, c)]: - route_cost_scaling = G.edges[(u, v, c)]['route_cost_scaling'] - else: - logger.warning( - "EDGE: {}, {}, {} - mode: {} - artificial {} -- " - "does not have key route_cost_scaling".format(u, v, c, mode_source, artificial)) - - edge_list.append( - [from_node_id, to_node_id, artificial, mode_source, mode_source_oid, miles, route_cost_scaling, - capacity, volume, vcr]) - - # the node_id will be used to explode the edges by commodity and time period - if edge_list: - update_sql = """ - INSERT into networkx_edges - values (null,?,?,?,?,?,?,?,?,?,?) - ;""" - # Add one more question mark here - db_con.executemany(update_sql, edge_list) - db_con.commit() - logger.debug("finished network_x edges commit") - - -# ---------------------------------------------------------------------------- - - -def read_shp(path, logger, simplify=True, geom_attrs=True, strict=True): - # the modified read_shp() multidigraph code - logger.debug("start: read_shp -- simplify: {}, geom_attrs: {}, strict: {}".format(simplify, geom_attrs, strict)) - - try: - from osgeo import ogr - except ImportError: - logger.error("read_shp requires OGR: http://www.gdal.org/") - raise ImportError("read_shp requires OGR: http://www.gdal.org/") - - if not isinstance(path, str): - return - net = nx.MultiDiGraph() - shp = ogr.Open(path) - if shp is None: - logger.error("Unable to open {}".format(path)) - raise RuntimeError("Unable to open {}".format(path)) - for lyr in shp: - count = lyr.GetFeatureCount() - logger.debug("processing layer: {} - feature_count: {} ".format(lyr.GetName(), count)) - - fields = [x.GetName() for x in lyr.schema] - logger.debug("f's in layer: {}".format(len(lyr))) - f_counter = 0 - time_counter_string = "" - for f in lyr: - - f_counter += 1 - if f_counter % 2000 == 0: - time_counter_string += ' .' - - if f_counter % 20000 == 0: - logger.debug("lyr: {} - feature counter: {} / {}".format(lyr.GetName(), f_counter, count)) - if f_counter == count: - logger.debug("lyr: {} - feature counter: {} / {}".format(lyr.GetName(), f_counter, count)) - logger.debug(time_counter_string + 'done.') - - g = f.geometry() - if g is None: - if strict: - logger.error("Bad data: feature missing geometry") - raise nx.NetworkXError("Bad data: feature missing geometry") - else: - continue - fld_data = [f.GetField(f.GetFieldIndex(x)) for x in fields] - attributes = dict(list(zip(fields, fld_data))) - attributes["ShpName"] = lyr.GetName() - # Note: Using layer level geometry type - if g.GetGeometryType() == ogr.wkbPoint: - net.add_node(g.GetPoint_2D(0), **attributes) - elif g.GetGeometryType() in (ogr.wkbLineString, - ogr.wkbMultiLineString): - for edge in edges_from_line(g, attributes, simplify, - geom_attrs): - e1, e2, attr = edge - net.add_edge(e1, e2) - key = len(list(net[e1][e2].keys())) - 1 - net[e1][e2][key].update(attr) - else: - if strict: - logger.error("GeometryType {} not supported". - format(g.GetGeometryType())) - raise nx.NetworkXError("GeometryType {} not supported". - format(g.GetGeometryType())) - - return net - - -# ---------------------------------------------------------------------------- - - -def edges_from_line(geom, attrs, simplify=True, geom_attrs=True): - """ - Generate edges for each line in geom - Written as a helper for read_shp - - Parameters - ---------- - - geom: ogr line geometry - To be converted into an edge or edges - - attrs: dict - Attributes to be associated with all geoms - - simplify: bool - If True, simplify the line as in read_shp - - geom_attrs: bool - If True, add geom attributes to edge as in read_shp - - - Returns - ------- - edges: generator of edges - each edge is a tuple of form - (node1_coord, node2_coord, attribute_dict) - suitable for expanding into a networkx Graph add_edge call - """ - try: - from osgeo import ogr - except ImportError: - raise ImportError("edges_from_line requires OGR: http://www.gdal.org/") - - if geom.GetGeometryType() == ogr.wkbLineString: - if simplify: - edge_attrs = attrs.copy() - last = geom.GetPointCount() - 1 - if geom_attrs: - edge_attrs["Wkb"] = geom.ExportToWkb() - edge_attrs["Wkt"] = geom.ExportToWkt() - edge_attrs["Json"] = geom.ExportToJson() - yield (geom.GetPoint_2D(0), geom.GetPoint_2D(last), edge_attrs) - else: - for i in range(0, geom.GetPointCount() - 1): - pt1 = geom.GetPoint_2D(i) - pt2 = geom.GetPoint_2D(i + 1) - edge_attrs = attrs.copy() - if geom_attrs: - segment = ogr.Geometry(ogr.wkbLineString) - segment.AddPoint_2D(pt1[0], pt1[1]) - segment.AddPoint_2D(pt2[0], pt2[1]) - edge_attrs["Wkb"] = segment.ExportToWkb() - edge_attrs["Wkt"] = segment.ExportToWkt() - edge_attrs["Json"] = segment.ExportToJson() - del segment - yield (pt1, pt2, edge_attrs) - - elif geom.GetGeometryType() == ogr.wkbMultiLineString: - for i in range(geom.GetGeometryCount()): - geom_i = geom.GetGeometryRef(i) - for edge in edges_from_line(geom_i, attrs, simplify, geom_attrs): - yield edge - +# ------------------------------------------------------------------------------ +# Name: ftot_networkx.py +# +# Purpose: the purpose of this module is to handle all the NetworkX methods and operations +# necessary to go between FTOT layers: GIS, sqlite DB, etc. +# Revised: 6/15/21 +# ------------------------------------------------------------------------------ + +import networkx as nx +import sqlite3 +from shutil import rmtree +import datetime +import ftot_supporting +import arcpy +import os +import multiprocessing +import math +from ftot_pulp import commodity_mode_setup + + +# ----------------------------------------------------------------------------- + + +def graph(the_scenario, logger): + + # check for permitted modes before creating nX graph + check_permitted_modes(the_scenario, logger) + + # export the assets from GIS export_fcs_from_main_gdb + export_fcs_from_main_gdb(the_scenario, logger) + + # create the networkx multidigraph + G = make_networkx_graph(the_scenario, logger) + + # clean up the networkx graph to preserve connectivity + clean_networkx_graph(the_scenario, G, logger) + + # cache the digraph to the db and store the route_cost_scaling factor + digraph_to_db(the_scenario, G, logger) + + # cost the network in the db + set_network_costs_in_db(the_scenario, logger) + + # generate shortest paths through the network + presolve_network(the_scenario, G, logger) + + # eliminate the graph and related shape files before moving on + delete_shape_files(the_scenario, logger) + G = nx.null_graph() + + +# ----------------------------------------------------------------------------- + + +def delete_shape_files(the_scenario, logger): + # delete temporary files + logger.debug("start: delete the temp_networkx_shp_files dir") + input_path = the_scenario.networkx_files_dir + rmtree(input_path) + logger.debug("finish: delete the temp_networkx_shp_files dir") + + +# ----------------------------------------------------------------------------- + + +# Scan the XML and input_data to ensure that pipelines are permitted and relevant +def check_permitted_modes(the_scenario, logger): + logger.debug("start: check permitted modes") + commodity_mode_dict = commodity_mode_setup(the_scenario, logger) + with sqlite3.connect(the_scenario.main_db) as db_cur: + # get pipeline records with an allow_yn == y + sql = "select * from commodity_mode where mode like 'pipeline%' and allowed_yn like 'y';" + pipeline_allowed = db_cur.execute(sql).fetchall() + if not pipeline_allowed: + logger.info("pipelines are not allowed") + new_permitted_mode_list = [] + for mode in the_scenario.permittedModes: + if 'pipeline' not in mode: + new_permitted_mode_list.append(mode) + elif 'pipeline' in mode: + continue # we don't want to include pipeline fcs if no csv exists to specify product flow + the_scenario.permittedModes = new_permitted_mode_list + logger.debug("finish: check permitted modes") + + +# ----------------------------------------------------------------------------- + + +def make_networkx_graph(the_scenario, logger): + # High level work flow: + # ------------------------ + # make_networkx_graph + # create the multidigraph + # convert the node labels to integers + # reverse the graph and compose with self + + logger.info("start: make_networkx_graph") + start_time = datetime.datetime.now() + + # read the shapefiles in the customized read_shp method + input_path = the_scenario.networkx_files_dir + + logger.debug("start: read_shp") + G = read_shp(input_path, logger, simplify=True, + geom_attrs=False, strict=True) # note this is custom and not nx.read_shp() + + # cleanup the node labels + logger.debug("start: convert node labels") + G = nx.convert_node_labels_to_integers(G, first_label=0, ordering='default', label_attribute="x_y_location") + + # create a reversed graph + logger.debug("start: reverse G graph to H") + H = G.reverse() # this is a reversed version of the graph. + + # set a new attribute for every edge that says its a "reversed" link + # we will use this to delete edges that shouldn't be reversed later. + logger.debug("start: set 'reversed' attribute in H") + nx.set_edge_attributes(H, 1, "REVERSED") + + # add the two graphs together + logger.debug("start: compose G and H") + G = nx.compose(G, H) + + # print out some stats on the Graph + logger.info("Number of nodes in the raw graph: {}".format(G.order())) + logger.info("Number of edges in the raw graph: {}".format(G.size())) + + logger.debug( + "finished: make_networkx_graph: Runtime (HMS): \t{}".format( + ftot_supporting.get_total_runtime_string(start_time))) + + return G + + +# ----------------------------------------------------------------------------- + + +def presolve_network(the_scenario, G, logger): + logger.debug("start: presolve_network") + + # Check NDR conditions before calculating shortest paths + update_ndr_parameter(the_scenario, logger) + + # Create a table to hold the shortest edges + with sqlite3.connect(the_scenario.main_db) as db_cur: + # clean up the db + sql = "drop table if exists shortest_edges;" + db_cur.execute(sql) + + sql = """ + create table if not exists shortest_edges (from_node_id INT, to_node_id INT, edge_id INT, + CONSTRAINT unique_from_to_edge_id_tuple UNIQUE (from_node_id, to_node_id, edge_id)); + """ + db_cur.execute(sql) + + # Create a table to hold routes + with sqlite3.connect(the_scenario.main_db) as db_cur: + # clean up the db + sql = "drop table if exists route_edges;" + db_cur.execute(sql) + + sql = """ + create table if not exists route_edges (from_node_id INT, to_node_id INT, edge_id INT, + scenario_rt_id INT, rt_order_ind INT); + """ + db_cur.execute(sql) + + # If NDR_On = False (default case), then skip calculating shortest paths + if not the_scenario.ndrOn: + with sqlite3.connect(the_scenario.main_db) as db_cur: + sql = """ + insert or ignore into shortest_edges + select from_node_id, to_node_id, edge_id + from networkx_edges; + """ + logger.debug("NDR skipped and all edges added to shortest_edges table") + db_cur.execute(sql) + db_cur.commit() + logger.debug("finish: presolve_network") + return + + # Get phases_of_matter in the scenario + phases_of_matter_in_scenario = get_phases_of_matter_in_scenario(the_scenario, logger) + + # Determine the weights for each phase of matter in scenario associated with the edges in the nX graph + nx_graph_weighting(the_scenario, G, phases_of_matter_in_scenario, logger) + + # Make subgraphs for combination of permitted modes + commodity_subgraph_dict = make_mode_subgraphs(the_scenario, G, logger) + + # Create a dictionary of edge_ids from the database which is used later to uniquely identify edges + edge_id_dict = find_edge_ids(the_scenario, logger) + + # Make origin-destination pairs where od_pairs is a dictionary keyed off [commodity_id, target, source], + # value is a list of [scenario_rt_id, phase_of_matter] for that key + od_pairs = make_od_pairs(the_scenario, logger) + + # Use multi-processing to determine shortest_paths for each target in the od_pairs dictionary + manager = multiprocessing.Manager() + all_route_edges = manager.list() + no_path_pairs = manager.list() + logger.debug("multiprocessing.cpu_count() = {}".format(multiprocessing.cpu_count())) + + # To parallelize computations, assign a set of targets to be passed to each processor + stuff_to_pass = [] + logger.debug("start: identify shortest_path between each o-d pair by commodity") + # by commodity and destination + for commodity_id in od_pairs: + phase_of_matter = od_pairs[commodity_id]['phase_of_matter'] + allowed_modes = commodity_subgraph_dict[commodity_id]['modes'] + for a_target in od_pairs[commodity_id]['targets'].keys(): + stuff_to_pass.append([commodity_subgraph_dict[commodity_id]['subgraph'], + od_pairs[commodity_id]['targets'][a_target], + a_target, all_route_edges, no_path_pairs, + edge_id_dict, phase_of_matter, allowed_modes]) + + # Allow multiprocessing, with no more than 75% of cores to be used, rounding down if necessary + logger.info("start: the multiprocessing route solve.") + processors_to_save = int(math.ceil(multiprocessing.cpu_count() * 0.25)) + processors_to_use = multiprocessing.cpu_count() - processors_to_save + logger.info("number of CPUs to use = {}".format(processors_to_use)) + + pool = multiprocessing.Pool(processes=processors_to_use) + try: + pool.map(multi_shortest_paths, stuff_to_pass) + except Exception as e: + pool.close() + pool.terminate() + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + pool.close() + pool.join() + + logger.info("end: identify shortest_path between each o-d pair") + + # Log any origin-destination pairs without a shortest path + if no_path_pairs: + logger.warning("Cannot identify shortest paths for {} o-d pairs; see log file list".format(len(no_path_pairs))) + for i in no_path_pairs: + s, t, rt_id = i + logger.debug("Missing shortest path for source {}, target {}, scenario_route_id {}".format(s, t, rt_id)) + + with sqlite3.connect(the_scenario.main_db) as db_cur: + sql = """ + insert into route_edges + (from_node_id, to_node_id, edge_id, scenario_rt_id, rt_order_ind) + values (?,?,?,?,?); + """ + + logger.debug("start: update route_edges table") + db_cur.executemany(sql, all_route_edges) + db_cur.commit() + logger.debug("end: update route_edges table") + + with sqlite3.connect(the_scenario.main_db) as db_cur: + sql = """ + insert or ignore into shortest_edges + (from_node_id, to_node_id, edge_id) + select distinct from_node_id, to_node_id, edge_id + from route_edges; + """ + logger.debug("start: update shortest_edges table") + db_cur.execute(sql) + db_cur.commit() + logger.debug("end: update shortest_edges table") + + logger.debug("finish: presolve_network") + + +# ----------------------------------------------------------------------------- + + +# Ensure shortest paths not calculated in presence of candidate generation, capacity, max transport distance +def update_ndr_parameter(the_scenario, logger): + logger.debug("start: check NDR conditions") + new_ndr_parameter = the_scenario.ndrOn + + # if capacity is enabled, then skip NDR + if the_scenario.capacityOn: + new_ndr_parameter = False + logger.debug("NDR de-activated due to capacity enforcement") + + # if candidate generation is active, then skip NDR + if the_scenario.processors_candidate_slate_data != 'None': + new_ndr_parameter = False + logger.debug("NDR de-activated due to candidate generation") + + # if max_transport_distance field is used in commodities table, then skip NDR + with sqlite3.connect(the_scenario.main_db) as main_db_con: + # get count of commodities with a specified max_transport_distance + sql = "select count(commodity_id) from commodities where max_transport_distance is not null;" + db_cur = main_db_con.execute(sql) + count = db_cur.fetchone()[0] + if count > 0: + new_ndr_parameter = False + logger.debug("NDR de-activated due to use of max transport distance") + + the_scenario.ndrOn = new_ndr_parameter + logger.debug("finish: check NDR conditions") + + +# ----------------------------------------------------------------------------- + + +# This method uses a shortest_path algorithm from the nx library to flag edges in the +# network that are a part of the shortest path connecting an origin to a destination +# for each commodity +def multi_shortest_paths(stuff_to_pass): + global all_route_edges, no_path_pairs + G, sources, target, all_route_edges, no_path_pairs, edge_id_dict, phase_of_matter, allowed_modes = stuff_to_pass + t = target + + shortest_paths_to_t = nx.shortest_path(G, target=t, weight='{}_weight'.format(phase_of_matter)) + for a_source in sources: + s = a_source + # This accounts for when a_source may not be connected to t, + # as is the case when certain modes may not be permitted + if a_source not in shortest_paths_to_t: + for i in sources[a_source]: + rt_id = i + no_path_pairs.append((s, t, rt_id)) + continue + for i in sources[a_source]: + rt_id = i + for index, from_node in enumerate(shortest_paths_to_t[s]): + if index < (len(shortest_paths_to_t[s]) - 1): + to_node = shortest_paths_to_t[s][index + 1] + # find the correct edge_id on the shortest path + min_route_cost = 999999999 + min_edge_id = None + for j in edge_id_dict[to_node][from_node]: + edge_id, mode_source, route_cost = j + if mode_source in allowed_modes: + if route_cost < min_route_cost: + min_route_cost = route_cost + min_edge_id = edge_id + if min_edge_id is None: + error = """something went wrong finding the edge_id from node {} to node {} + for scenario_rt_id {} in shortest path algorithm""".format(from_node, to_node, rt_id) + raise Exception(error) + all_route_edges.append((from_node, to_node, min_edge_id, rt_id, index + 1)) + + +# ----------------------------------------------------------------------------- + + +# Assigns for each link in the networkX graph a weight for each phase of matter +# which mirrors the cost of each edge for the optimizer +def nx_graph_weighting(the_scenario, G, phases_of_matter_in_scenario, logger): + + # pull the route cost for all edges in the graph + logger.debug("start: assign edge weights to networkX graph") + for phase_of_matter in phases_of_matter_in_scenario: + # initialize the edge weight variable to something large to be overwritten in the loop + nx.set_edge_attributes(G, 999999999, name='{}_weight'.format(phase_of_matter)) + + for (u, v, c, d) in G.edges(keys=True, data='route_cost_scaling', default=False): + from_node_id = u + to_node_id = v + route_cost_scaling = G.edges[(u, v, c)]['route_cost_scaling'] + mileage = G.edges[(u, v, c)]['MILES'] + source = G.edges[(u, v, c)]['source'] + artificial = G.edges[(u, v, c)]['Artificial'] + + # calculate route_cost for all edges and phases of matter to mirror network costs in db + for phase_of_matter in phases_of_matter_in_scenario: + weight = get_network_link_cost(the_scenario, phase_of_matter, source, artificial, logger) + if 'pipeline' not in source: + if artificial == 0: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = mileage * route_cost_scaling * weight + elif artificial == 1: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight + elif artificial == 2: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight / 2.00 + else: + logger.warning("artificial code of {} is not supported!".format(artificial)) + else: + # inflate the cost of pipelines for 'solid' to avoid forming a shortest path with them + if phase_of_matter == 'solid': + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = 1000000 + else: + if artificial == 0: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = route_cost_scaling + elif artificial == 1: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = mileage * weight + elif artificial == 2: + G.edges[(u, v, c)]['{}_weight'.format(phase_of_matter)] = weight / 2.00 + else: + logger.warning("artificial code of {} is not supported!".format(artificial)) + + logger.debug("end: assign edge weights to networkX graph") + + +# ----------------------------------------------------------------------------- + + +# Returns a dictionary of NetworkX graphs keyed off commodity_id with 'modes' and 'subgraph' keys +def make_mode_subgraphs(the_scenario, G, logger): + logger.debug("start: create mode subgraph dictionary") + + logger.debug("start: pull commodity mode from SQL") + with sqlite3.connect(the_scenario.main_db) as db_cur: + sql = "select mode, commodity_id from commodity_mode where allowed_yn like 'y';" + commodity_mode_data = db_cur.execute(sql).fetchall() + commodity_subgraph_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + if commodity_id not in commodity_subgraph_dict: + commodity_subgraph_dict[commodity_id] = {} + commodity_subgraph_dict[commodity_id]['modes'] = [] + commodity_subgraph_dict[commodity_id]['modes'].append(mode) + logger.debug("end: pull commodity mode from SQL") + + for k in commodity_subgraph_dict: + commodity_subgraph_dict[k]['modes'] = sorted(commodity_subgraph_dict[k]['modes']) + + # check if commodity mode input file exists + if not os.path.exists(the_scenario.commodity_mode_data): + # all commodities are allowed on all permitted modes and use graph G + logger.debug("no commodity_mode_data file: {}".format(the_scenario.commodity_mode_data)) + logger.debug("all commodities allowed on all permitted modes") + for k in commodity_subgraph_dict: + commodity_subgraph_dict[k]['subgraph'] = G + else: + # generate subgraphs for each unique combination of allowed modes + logger.debug("creating subgraphs for each unique set of allowed modes") + # create a dictionary of subgraphs indexed by 'modes' + subgraph_dict = {} + for k in commodity_subgraph_dict: + allowed_modes = str(commodity_subgraph_dict[k]['modes']) + if allowed_modes not in subgraph_dict: + subgraph_dict[allowed_modes] = nx.DiGraph(((source, target, attr) for source, target, attr in + G.edges(data=True) if attr['MODE_TYPE'] in + commodity_subgraph_dict[k]['modes'])) + + # assign subgraphs to commodities + for k in commodity_subgraph_dict: + commodity_subgraph_dict[k]['subgraph'] = subgraph_dict[str(commodity_subgraph_dict[k]['modes'])] + + logger.debug("end: create mode subgraph dictionary") + + return commodity_subgraph_dict + + +# ----------------------------------------------------------------------------- + + +# Returns a dictionary of edge_ids keyed off (to_node_id, from_node_id) +def find_edge_ids(the_scenario, logger): + logger.debug("start: create edge_id dictionary") + logger.debug("start: pull edge_ids from SQL") + with sqlite3.connect(the_scenario.main_db) as db_cur: + sql = """ + select ne.from_node_id, ne.to_node_id, ne.edge_id, ne.mode_source, nec.route_cost + from networkx_edges ne + left join networkx_edge_costs nec + on ne.edge_id = nec.edge_id; + """ + sql_list = db_cur.execute(sql).fetchall() + logger.debug("end: pull edge_ids from SQL") + + edge_id_dict = {} + + for row in sql_list: + from_node = row[0] + to_node = row[1] + edge_id = row[2] + mode_source = row[3] + route_cost = row[4] + if to_node not in edge_id_dict: + edge_id_dict[to_node] = {} + if from_node not in edge_id_dict[to_node].keys(): + edge_id_dict[to_node][from_node] = [] + edge_id_dict[to_node][from_node].append([edge_id, mode_source, route_cost]) + + logger.debug("end: create edge_id dictionary") + + return edge_id_dict + + +# ----------------------------------------------------------------------------- + + +# Creates a dictionary of all feasible origin-destination pairs, including: +# RMP-DEST, RMP-PROC, PROC-DEST, etc., indexed by [commodity_id][target][source] +def make_od_pairs(the_scenario, logger): + with sqlite3.connect(the_scenario.main_db) as db_cur: + # Create a table for od_pairs in the database + logger.info("start: create o-d pairs table") + sql = "drop table if exists od_pairs;" + db_cur.execute(sql) + + sql = ''' + create table od_pairs(scenario_rt_id INTEGER PRIMARY KEY, from_location_id integer, to_location_id integer, + from_facility_id integer, to_facility_id integer, commodity_id integer, phase_of_matter text, route_status text, + from_node_id INTEGER, to_node_id INTEGER, from_location_1 INTEGER, to_location_1 INTEGER); + ''' + db_cur.execute(sql) + + # Populate the od_pairs table from a temporary table that collects both origins and destinations + sql = "drop table if exists tmp_connected_facilities_with_commodities;" + db_cur.execute(sql) + + sql = ''' + create table tmp_connected_facilities_with_commodities as + select + facilities.facility_id, + facilities.location_id, + facilities.facility_name, + facility_type_id.facility_type, + ignore_facility, + facility_commodities.commodity_id, + facility_commodities.io, + commodities.phase_of_matter, + networkx_nodes.node_id, + networkx_nodes.location_1 + from facilities + join facility_commodities on + facility_commodities.facility_id = facilities.facility_ID + join commodities on + facility_commodities.commodity_id = commodities.commodity_id + join facility_type_id on + facility_type_id.facility_type_id = facilities.facility_type_id + join networkx_nodes on + networkx_nodes.location_id = facilities.location_id + where ignore_facility = 'false'; + ''' + db_cur.execute(sql) + + sql = ''' + insert into od_pairs (from_location_id, to_location_id, from_facility_id, to_facility_id, commodity_id, + phase_of_matter, from_node_id, to_node_id, from_location_1, to_location_1) + select distinct + origin.location_id AS from_location_id, + destination.location_id AS to_location_id, + origin.facility_id AS from_facility_id, + destination.facility_id AS to_facility_id, + origin.commodity_id AS commodity_id, + origin.phase_of_matter AS phase_of_matter, + origin.node_id AS from_node_id, + destination.node_id AS to_node_id, + origin.location_1 AS from_location_1, + destination.location_1 AS to_location_1 + from + tmp_connected_facilities_with_commodities as origin + inner join + tmp_connected_facilities_with_commodities as destination ON + CASE + WHEN origin.facility_type <> 'processor' or destination.facility_type <> 'processor' -- THE NORMAL CASE, RMP->PROC, RMP->DEST, or PROC->DEST + THEN + origin.facility_type <> destination.facility_type -- not the same facility_type + and + origin.commodity_id = destination.commodity_id -- match on commodity + and + origin.facility_id <> destination.facility_id -- not the same facility + and + origin.facility_type <> "ultimate_destination" -- restrict origin types + and + destination.facility_type <> "raw_material_producer" -- restrict destination types + and + destination.facility_type <> "raw_material_producer_as_processor" -- restrict other destination types TODO - MNP - 12/6/17 MAY NOT NEED THIS IF WE MAKE TEMP CANDIDATES AS THE RMP + and + origin.location_1 like '%_OUT' -- restrict to the correct out/in node_id's + AND destination.location_1 like '%_IN' + ELSE -- THE CASE WHEN PROCESSORS ARE SENDING STUFF TO OTHER PROCESSORS + origin.io = 'o' -- make sure processors origins send outputs + and + destination.io = 'i' -- make sure processors destinations receive inputs + and + origin.commodity_id = destination.commodity_id -- match on commodity + and + origin.facility_id <> destination.facility_id -- not the same facility + and + origin.facility_type <> "ultimate_destination" -- restrict origin types + and + destination.facility_type <> "raw_material_producer" -- restrict destination types + and + destination.facility_type <> "raw_material_producer_as_processor" -- restrict other destination types TODO - MNP - 12/6/17 MAY NOT NEED THIS IF WE MAKE TEMP CANDIDATES AS THE RMP + and + origin.location_1 like '%_OUT' -- restrict to the correct out/in node_id's + AND destination.location_1 like '%_IN' + END; + ''' + db_cur.execute(sql) + + logger.debug("drop the tmp_connected_facilities_with_commodities table") + db_cur.execute("drop table if exists tmp_connected_facilities_with_commodities;") + + logger.info("end: create o-d pairs table") + + # Fetch all od-pairs, ordered by target + sql = ''' + SELECT to_node_id, from_node_id, scenario_rt_id, commodity_id, phase_of_matter + FROM od_pairs ORDER BY to_node_id DESC; + ''' + sql_list = db_cur.execute(sql).fetchall() + + # Loop through the od_pairs + od_pairs = {} + for row in sql_list: + target = row[0] + source = row[1] + scenario_rt_id = row[2] + commodity_id = row[3] + phase_of_matter = row[4] + if commodity_id not in od_pairs: + od_pairs[commodity_id] = {} + od_pairs[commodity_id]['phase_of_matter'] = phase_of_matter + od_pairs[commodity_id]['targets'] = {} + if target not in od_pairs[commodity_id]['targets'].keys(): + od_pairs[commodity_id]['targets'][target] = {} + if source not in od_pairs[commodity_id]['targets'][target].keys(): + od_pairs[commodity_id]['targets'][target][source] = [] + od_pairs[commodity_id]['targets'][target][source].append(scenario_rt_id) + + return od_pairs + + +# ----------------------------------------------------------------------------- + + +def export_fcs_from_main_gdb(the_scenario, logger): + # export fcs from the main.GDB to individual shapefiles + logger.info("start: export_fcs_from_main_gdb") + start_time = datetime.datetime.now() + + # export network and locations fc's to shapefiles + main_gdb = the_scenario.main_gdb + output_path = the_scenario.networkx_files_dir + input_features = [] + + logger.debug("start: create temp_networkx_shp_files dir") + if os.path.exists(output_path): + logger.debug("deleting pre-existing temp_networkx_shp_files dir") + rmtree(output_path) + + if not os.path.exists(output_path): + logger.debug("creating new temp_networkx_shp_files dir") + os.makedirs(output_path) + + location_list = ['\\locations', '\\network\\intermodal', '\\network\\locks'] + + # only add shape files associated with modes that are permitted in the scenario file + for mode in the_scenario.permittedModes: + location_list.append('\\network\\{}'.format(mode)) + + # get the locations and network feature layers + for fc in location_list: + input_features.append(main_gdb + fc) + arcpy.FeatureClassToShapefile_conversion(Input_Features=input_features, Output_Folder=output_path) + + logger.debug("finished: export_fcs_from_main_gdb: Runtime (HMS): \t{}".format( + ftot_supporting.get_total_runtime_string(start_time))) + + +# ------------------------------------------------------------------------------ + + +def clean_networkx_graph(the_scenario, G, logger): + # ------------------------------------------------------------------------- + # renamed clean_networkx_graph () + # remove reversed links for pipeline + # selectively remove links for location _IN and _OUT nodes + # preserve the route_cost_scaling factor in an attribute by phase of matter + + logger.info("start: clean_networkx_graph") + start_time = datetime.datetime.now() + + logger.debug("Processing the {} edges in the uncosted graph.".format(G.size())) + + # use the artificial and reversed attribute to determine if + # the link is kept + # ------------------------------------------------------------- + edge_attrs = {} # for storing the edge attributes which are set all at once + deleted_edge_count = 0 + + for u, v, keys, artificial in list(G.edges(data='Artificial', keys=True)): + + # initialize the route_cost_scaling variable to something + # absurd so we know if its getting set properly in the loop: + route_cost_scaling = -999999999 + + # check if the link is reversed + if 'REVERSED' in G.edges[u, v, keys]: + reversed_link = G.edges[u, v, keys]['REVERSED'] + else: + reversed_link = 0 + + # check if capacity is 0 - not currently considered here + # Network Edges - artificial == 0 + # ----------------------------------- + if artificial == 0: + + # check the mode type + # ---------------------- + mode_type = G.edges[u, v, keys]['MODE_TYPE'] + + # set the mode specific weights + # ----------------------------- + + if mode_type == "rail": + d_code = G.edges[u, v, keys]["DENSITY_CO"] + if d_code in [7]: + route_cost_scaling = the_scenario.rail_dc_7 + elif d_code in [6]: + route_cost_scaling = the_scenario.rail_dc_6 + elif d_code in [5]: + route_cost_scaling = the_scenario.rail_dc_5 + elif d_code in [4]: + route_cost_scaling = the_scenario.rail_dc_4 + elif d_code in [3]: + route_cost_scaling = the_scenario.rail_dc_3 + elif d_code in [2]: + route_cost_scaling = the_scenario.rail_dc_2 + elif d_code in [1]: + route_cost_scaling = the_scenario.rail_dc_1 + elif d_code in [0]: + route_cost_scaling = the_scenario.rail_dc_0 + else: + logger.warning("The d_code {} is not supported".format(d_code)) + + elif mode_type == "water": + + # get the total vol of water traffic + tot_vol = G.edges[u, v, keys]['TOT_UP_DWN'] + if tot_vol >= 10000000: + route_cost_scaling = the_scenario.water_high_vol + elif 1000000 <= tot_vol < 10000000: + route_cost_scaling = the_scenario.water_med_vol + elif 1 <= tot_vol < 1000000: + route_cost_scaling = the_scenario.water_low_vol + else: + route_cost_scaling = the_scenario.water_no_vol + + elif mode_type == "road": + + # get fclass + fclass = G.edges[u, v, keys]['FCLASS'] + if fclass in [1]: + route_cost_scaling = the_scenario.truck_interstate + elif fclass in [2, 3]: + route_cost_scaling = the_scenario.truck_pr_art + elif fclass in [4]: + route_cost_scaling = the_scenario.truck_m_art + else: + route_cost_scaling = the_scenario.truck_local + + elif 'pipeline' in mode_type: + if reversed_link == 1: + G.remove_edge(u, v, keys) + deleted_edge_count += 1 + continue # move on to the next edge + else: + route_cost_scaling = (((float(G.edges[u, v, keys]['base_rate']) / 100) / 42.0) * 1000.0) + + # Intermodal Edges - artificial == 2 + # ------------------------------------ + elif artificial == 2: + # set it to 1 because we'll multiply by the appropriate + # link_cost later for transloading + route_cost_scaling = 1 + + # Artificial Edge - artificial == 1 + # ---------------------------------- + # need to check if its an IN location or an OUT location and delete selectively. + # assume always connecting from the node to the network. + # so _OUT locations should delete the reversed link + # _IN locations should delete the non-reversed link. + elif artificial == 1: + # delete edges we dont want + + try: + if G.edges[u, v, keys]['LOCATION_1'].find("_OUT") > -1 and reversed_link == 1: + G.remove_edge(u, v, keys) + deleted_edge_count += 1 + continue # move on to the next edge + elif G.edges[u, v, keys]['LOCATION_1'].find("_IN") > -1 and reversed_link == 0: + G.remove_edge(u, v, keys) + deleted_edge_count += 1 + continue # move on to the next edge + + # there is no scaling of artificial links. + # the cost_penalty is calculated in get_network_link_cost() + else: + route_cost_scaling = 1 + except: + logger.warning("the following keys didn't work:u - {}, v- {}".format(u, v)) + else: + logger.warning("found an edge without artificial attribute: {} ") + continue + + edge_attrs[u, v, keys] = { + 'route_cost_scaling': route_cost_scaling + } + + nx.set_edge_attributes(G, edge_attrs) + + # print out some stats on the Graph + logger.info("Number of nodes in the clean graph: {}".format(G.order())) + logger.info("Number of edges in the clean graph: {}".format(G.size())) + + logger.debug("finished: clean_networkx_graph: Runtime (HMS): \t{}".format( + ftot_supporting.get_total_runtime_string(start_time))) + + return G + + +# ------------------------------------------------------------------------------ + + +def get_network_link_cost(the_scenario, phase_of_matter, mode, artificial, logger): + # three types of artificial links: + # (0 = network edge, 2 = intermodal, 1 = artificial link btw facility location and network edge) + # add the appropriate cost to the network edges based on phase of matter + + if phase_of_matter == "solid": + # set the mode costs + truck_base_cost = the_scenario.solid_truck_base_cost.magnitude + railroad_class_1_cost = the_scenario.solid_railroad_class_1_cost.magnitude + barge_cost = the_scenario.solid_barge_cost.magnitude + transloading_cost = the_scenario.solid_transloading_cost.magnitude + rail_short_haul_penalty = the_scenario.solid_rail_short_haul_penalty.magnitude + water_short_haul_penalty = the_scenario.solid_water_short_haul_penalty.magnitude + + elif phase_of_matter == "liquid": + # set the mode costs + truck_base_cost = the_scenario.liquid_truck_base_cost.magnitude + railroad_class_1_cost = the_scenario.liquid_railroad_class_1_cost.magnitude + barge_cost = the_scenario.liquid_barge_cost.magnitude + transloading_cost = the_scenario.liquid_transloading_cost.magnitude + rail_short_haul_penalty = the_scenario.liquid_rail_short_haul_penalty.magnitude + water_short_haul_penalty = the_scenario.liquid_water_short_haul_penalty.magnitude + + else: + logger.error("the phase of matter: -- {} -- is not supported. returning") + raise NotImplementedError + + if artificial == 1: + # add a fixed cost penalty to the routing cost for artificial links + if mode == "rail": + link_cost = rail_short_haul_penalty / 2.0 + # Divide by 2 is to ensure the penalty is not doubled-- it is applied on artificial links on both ends + elif mode == "water": + link_cost = water_short_haul_penalty / 2.0 + # Divide by 2 is to ensure the penalty is not doubled-- it is applied on artificial links on both ends + elif 'pipeline' in mode: + # this cost penalty was calculated by looking at the average per mile base rate. + link_cost = 0.19 + # no mileage multiplier here for pipeline as unlike rail/water, we do not want to disproportionally + # discourage short movements + # Multiplier will be applied based on actual link mileage when scenario costs are actually set + else: + link_cost = the_scenario.truck_local * truck_base_cost # for road + + elif artificial == 2: + # phase of mater is determined above + link_cost = transloading_cost + + elif artificial == 0: + if mode == "road": + link_cost = truck_base_cost + elif mode == "rail": + link_cost = railroad_class_1_cost + elif mode == "water": + link_cost = barge_cost + elif mode == "pipeline_crude_trf_rts": + link_cost = 1 # so we multiply by the base_rate + elif mode == "pipeline_prod_trf_rts": + link_cost = 1 # so we multiply by base_rate + + return link_cost + + +# ---------------------------------------------------------------------------- + + +def get_phases_of_matter_in_scenario(the_scenario, logger): + logger.debug("start: get_phases_of_matter_in_scenario()") + + phases_of_matter_in_scenario = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + sql = "select count(distinct phase_of_matter) from commodities where phase_of_matter is not null;" + db_cur = main_db_con.execute(sql) + + count = db_cur.fetchone()[0] + logger.debug("phases_of_matter in the scenario: {}".format(count)) + + if not count: + error = "No phases of matter in the scenario: {}...returning".format(count) + logger.error(error) + raise Exception(error) + + elif count: + sql = "select phase_of_matter from commodities where phase_of_matter is not null group by phase_of_matter" + db_cur = main_db_con.execute(sql) + for row in db_cur: + phases_of_matter_in_scenario.append(row[0]) + else: + logger.warning("Something went wrong in get_phases_of_matter_in_scenario()") + error = "Count phases of matter to route: {}".format(str(count)) + logger.error(error) + raise Exception(error) + + logger.debug("end: get_phases_of_matter_in_scenario()") + return phases_of_matter_in_scenario + + +# ----------------------------------------------------------------------------- + + +# set the network costs in the db by phase_of_matter +def set_network_costs_in_db(the_scenario, logger): + + logger.info("start: set_network_costs_in_db") + with sqlite3.connect(the_scenario.main_db) as db_con: + # clean up the db + sql = "drop table if exists networkx_edge_costs" + db_con.execute(sql) + + sql = "create table if not exists networkx_edge_costs " \ + "(edge_id INTEGER, phase_of_matter_id INT, route_cost REAL, dollar_cost REAL)" + db_con.execute(sql) + + # build up the network edges cost by phase of matter + edge_cost_list = [] + + # get phases_of_matter in the scenario + phases_of_matter_in_scenario = get_phases_of_matter_in_scenario(the_scenario, logger) + + # loop through each edge in the networkx_edges table + sql = "select edge_id, mode_source, artificial, miles, route_cost_scaling from networkx_edges" + db_cur = db_con.execute(sql) + for row in db_cur: + + edge_id = row[0] + mode_source = row[1] + artificial = row[2] + miles = row[3] + route_cost_scaling = row[4] + + for phase_of_matter in phases_of_matter_in_scenario: + + # skip pipeline and solid phase of matter + if phase_of_matter == 'solid' and 'pipeline' in mode_source: + continue + + # otherwise, go ahead and get the link cost + link_cost = get_network_link_cost(the_scenario, phase_of_matter, mode_source, artificial, logger) + + if artificial == 0: + # road, rail, and water + if 'pipeline' not in mode_source: + dollar_cost = miles * link_cost # link_cost is taken from the scenario file + route_cost = dollar_cost * route_cost_scaling # this includes impedance + + else: + # if artificial = 0, route_cost_scaling = base rate + # we use the route_cost_scaling for this since its set in the GIS + # not in the scenario xml file. + + dollar_cost = route_cost_scaling # this is the base rate + route_cost = route_cost_scaling # this is the base rate + + elif artificial == 1: + # we don't want to add the cost penalty to the dollar cost for artificial links + dollar_cost = 0 + + if 'pipeline' not in mode_source: + # the routing_cost should have the artificial link penalty + # and artificial link penalties shouldn't be scaled by mileage. + route_cost = link_cost + + else: + # For pipeline we don't want to penalize short movements disproportionately, + # so scale penalty by miles. This gives art links for pipeline a modest routing cost + route_cost = link_cost * miles + + elif artificial == 2: + dollar_cost = link_cost / 2.00 # this is the transloading fee + # For now dividing by 2 to ensure that the transloading fee is not applied twice + # (e.g. on way in and on way out) + route_cost = link_cost / 2.00 # same as above. + + else: + logger.warning("artificial code of {} is not supported!".format(artificial)) + + edge_cost_list.append([edge_id, phase_of_matter, route_cost, dollar_cost]) + + if edge_cost_list: + update_sql = """ + INSERT into networkx_edge_costs + values (?,?,?,?) + ;""" + + db_con.executemany(update_sql, edge_cost_list) + logger.debug("start: networkx_edge_costs commit") + db_con.commit() + logger.debug("finish: networkx_edge_costs commit") + + logger.debug("finished: set_network_costs_in_db") + + +# ----------------------------------------------------------------------------- + + +def digraph_to_db(the_scenario, G, logger): + # moves the networkX digraph into the database for the pulp handshake + + logger.info("start: digraph_to_db") + with sqlite3.connect(the_scenario.main_db) as db_con: + + # clean up the db + sql = "drop table if exists networkx_nodes" + db_con.execute(sql) + + sql = "create table if not exists networkx_nodes (node_id INT, source TEXT, source_OID integer, location_1 " \ + "TEXT, location_id TEXT, shape_x REAL, shape_y REAL)" + db_con.execute(sql) + + # loop through the nodes in the digraph and set them in the db + # nodes will be either locations (with a location_id), or nodes connecting + # network edges (with no location info). + node_list = [] + + for node in G.nodes(): + source = None + source_oid = None + location_1 = None + location_id = None + shape_x = None + shape_y = None + + if 'source' in G.nodes[node]: + source = G.nodes[node]['source'] + + if 'source_OID' in G.nodes[node]: + source_oid = G.nodes[node]['source_OID'] + + if 'location_1' in G.nodes[node]: # for locations + location_1 = G.nodes[node]['location_1'] + location_id = G.nodes[node]['location_i'] + + if 'x_y_location' in G.nodes[node]: + shape_x = G.nodes[node]['x_y_location'][0] + shape_y = G.nodes[node]['x_y_location'][1] + + node_list.append([node, source, source_oid, location_1, location_id, shape_x, shape_y]) + + if node_list: + update_sql = """ + INSERT into networkx_nodes + values (?,?,?,?,?,?,?) + ;""" + + db_con.executemany(update_sql, node_list) + db_con.commit() + logger.debug("finished network_x nodes commit") + + # loop through the edges in the digraph and insert them into the db. + # ------------------------------------------------------------------- + edge_list = [] + with sqlite3.connect(the_scenario.main_db) as db_con: + + # clean up the db + sql = "drop table if exists networkx_edges" + db_con.execute(sql) + + sql = "create table if not exists networkx_edges (edge_id INTEGER PRIMARY KEY, from_node_id INT, to_node_id " \ + "INT, artificial INT, mode_source TEXT, mode_source_oid INT, miles REAL, route_cost_scaling REAL, " \ + "capacity INT, volume REAL, VCR REAL)" + db_con.execute(sql) + + for (u, v, c, d) in G.edges(keys=True, data='route_cost_scaling', default=False): + from_node_id = u + to_node_id = v + miles = G.edges[(u, v, c)]['MILES'] + artificial = G.edges[(u, v, c)]['Artificial'] + mode_source = G.edges[(u, v, c)]['MODE_TYPE'] + mode_source_oid = G.edges[(u, v, c)]['source_OID'] + + if mode_source in ['rail', 'road']: + volume = G.edges[(u, v, c)]['Volume'] + vcr = G.edges[(u, v, c)]['VCR'] + capacity = G.edges[(u, v, c)]['Capacity'] + else: + volume = None + vcr = None + capacity = None + + if capacity == 0: + capacity = None + logger.detailed_debug("link capacity == 0, setting to None".format(G.edges[(u, v, c)])) + + if 'route_cost_scaling' in G.edges[(u, v, c)]: + route_cost_scaling = G.edges[(u, v, c)]['route_cost_scaling'] + else: + logger.warning( + "EDGE: {}, {}, {} - mode: {} - artificial {} -- " + "does not have key route_cost_scaling".format(u, v, c, mode_source, artificial)) + + edge_list.append( + [from_node_id, to_node_id, artificial, mode_source, mode_source_oid, miles, route_cost_scaling, + capacity, volume, vcr]) + + # the node_id will be used to explode the edges by commodity and time period + if edge_list: + update_sql = """ + INSERT into networkx_edges + values (null,?,?,?,?,?,?,?,?,?,?) + ;""" + # Add one more question mark here + db_con.executemany(update_sql, edge_list) + db_con.commit() + logger.debug("finished network_x edges commit") + + +# ---------------------------------------------------------------------------- + + +def read_shp(path, logger, simplify=True, geom_attrs=True, strict=True): + # the modified read_shp() multidigraph code + logger.debug("start: read_shp -- simplify: {}, geom_attrs: {}, strict: {}".format(simplify, geom_attrs, strict)) + + try: + from osgeo import ogr + except ImportError: + logger.error("read_shp requires OGR: http://www.gdal.org/") + raise ImportError("read_shp requires OGR: http://www.gdal.org/") + + if not isinstance(path, str): + return + net = nx.MultiDiGraph() + shp = ogr.Open(path) + if shp is None: + logger.error("Unable to open {}".format(path)) + raise RuntimeError("Unable to open {}".format(path)) + for lyr in shp: + count = lyr.GetFeatureCount() + logger.debug("processing layer: {} - feature_count: {} ".format(lyr.GetName(), count)) + + fields = [x.GetName() for x in lyr.schema] + logger.debug("f's in layer: {}".format(len(lyr))) + f_counter = 0 + time_counter_string = "" + for f in lyr: + + f_counter += 1 + if f_counter % 2000 == 0: + time_counter_string += ' .' + + if f_counter % 20000 == 0: + logger.debug("lyr: {} - feature counter: {} / {}".format(lyr.GetName(), f_counter, count)) + if f_counter == count: + logger.debug("lyr: {} - feature counter: {} / {}".format(lyr.GetName(), f_counter, count)) + logger.debug(time_counter_string + 'done.') + + g = f.geometry() + if g is None: + if strict: + logger.error("Bad data: feature missing geometry") + raise nx.NetworkXError("Bad data: feature missing geometry") + else: + continue + fld_data = [f.GetField(f.GetFieldIndex(x)) for x in fields] + attributes = dict(list(zip(fields, fld_data))) + attributes["ShpName"] = lyr.GetName() + # Note: Using layer level geometry type + if g.GetGeometryType() == ogr.wkbPoint: + net.add_node(g.GetPoint_2D(0), **attributes) + elif g.GetGeometryType() in (ogr.wkbLineString, + ogr.wkbMultiLineString): + for edge in edges_from_line(g, attributes, simplify, + geom_attrs): + e1, e2, attr = edge + net.add_edge(e1, e2) + key = len(list(net[e1][e2].keys())) - 1 + net[e1][e2][key].update(attr) + else: + if strict: + logger.error("GeometryType {} not supported". + format(g.GetGeometryType())) + raise nx.NetworkXError("GeometryType {} not supported". + format(g.GetGeometryType())) + + return net + + +# ---------------------------------------------------------------------------- + + +def edges_from_line(geom, attrs, simplify=True, geom_attrs=True): + """ + Generate edges for each line in geom + Written as a helper for read_shp + + Parameters + ---------- + + geom: ogr line geometry + To be converted into an edge or edges + + attrs: dict + Attributes to be associated with all geoms + + simplify: bool + If True, simplify the line as in read_shp + + geom_attrs: bool + If True, add geom attributes to edge as in read_shp + + + Returns + ------- + edges: generator of edges + each edge is a tuple of form + (node1_coord, node2_coord, attribute_dict) + suitable for expanding into a networkx Graph add_edge call + """ + try: + from osgeo import ogr + except ImportError: + raise ImportError("edges_from_line requires OGR: http://www.gdal.org/") + + if geom.GetGeometryType() == ogr.wkbLineString: + if simplify: + edge_attrs = attrs.copy() + last = geom.GetPointCount() - 1 + if geom_attrs: + edge_attrs["Wkb"] = geom.ExportToWkb() + edge_attrs["Wkt"] = geom.ExportToWkt() + edge_attrs["Json"] = geom.ExportToJson() + yield (geom.GetPoint_2D(0), geom.GetPoint_2D(last), edge_attrs) + else: + for i in range(0, geom.GetPointCount() - 1): + pt1 = geom.GetPoint_2D(i) + pt2 = geom.GetPoint_2D(i + 1) + edge_attrs = attrs.copy() + if geom_attrs: + segment = ogr.Geometry(ogr.wkbLineString) + segment.AddPoint_2D(pt1[0], pt1[1]) + segment.AddPoint_2D(pt2[0], pt2[1]) + edge_attrs["Wkb"] = segment.ExportToWkb() + edge_attrs["Wkt"] = segment.ExportToWkt() + edge_attrs["Json"] = segment.ExportToJson() + del segment + yield (pt1, pt2, edge_attrs) + + elif geom.GetGeometryType() == ogr.wkbMultiLineString: + for i in range(geom.GetGeometryCount()): + geom_i = geom.GetGeometryRef(i) + for edge in edges_from_line(geom_i, attrs, simplify, geom_attrs): + yield edge + diff --git a/program/ftot_postprocess.py b/program/ftot_postprocess.py index f9a0c71..2b36055 100644 --- a/program/ftot_postprocess.py +++ b/program/ftot_postprocess.py @@ -1,1844 +1,1961 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot_postprocess.py -# -# Purpose: Post processes the optimal results, generates optimal segments FC and -# optimal_scenario_results table in the database -# -# --------------------------------------------------------------------------------------------------- - -import ftot_supporting_gis -import arcpy -import sqlite3 -import os -from collections import defaultdict - - -# ========================================================================= - - -def route_post_optimization_db(the_scenario, logger): - - logger.info("starting route_post_optimization_db") - - # Parse the Optimal Solution from the DB - # --------------------------------------- - from ftot_pulp import parse_optimal_solution_db - parsed_optimal_solution = parse_optimal_solution_db(the_scenario, logger) - optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material = parsed_optimal_solution - - if not the_scenario.ndrOn: - # Make the optimal routes and route_segments FCs from the db - # ------------------------------------------------------------ - make_optimal_route_segments_db(the_scenario, logger) - elif the_scenario.ndrOn: - # Make the optimal routes and route_segments FCs from the db - # ------------------------------------------------------------ - make_optimal_route_segments_from_routes_db(the_scenario, logger) - - # Make the optimal routes and route segments FCs - # ----------------------------------------------- - # One record for each link in the optimal flow by commodity - make_optimal_route_segments_featureclass_from_db(the_scenario, logger) - - # Add functional class and urban code to the road segments - add_fclass_and_urban_code(the_scenario, logger) - - dissolve_optimal_route_segments_feature_class_for_mapping(the_scenario, logger) - - make_optimal_scenario_results_db(the_scenario, logger) - - # Report the utilization by commodity - # ------------------------------------ - db_report_commodity_utilization(the_scenario, logger) - - # Generate the scenario summary - # ------------------------------ - generate_scenario_summary(the_scenario, logger) - - # Generate the artificial link summary - # ------------------------------ - generate_artificial_link_summary(the_scenario, logger) - - # Make the optimal intermodal facilities - # ---------------------------- - if not the_scenario.ndrOn: - make_optimal_intermodal_db(the_scenario, logger) - elif the_scenario.ndrOn: - make_optimal_intermodal_from_routes_db(the_scenario, logger) - - make_optimal_intermodal_featureclass(the_scenario, logger) - - # Make the optimal facilities - # ---------------------------- - make_optimal_facilities_db(the_scenario, logger) - - # Now that we have a table with the optimal facilities, we can go through each FC and see if the name - # matches an "optimal" facility_name, and set the flag. - - # -- RMP fc and reporting - make_optimal_raw_material_producer_featureclass(the_scenario, logger) - - # -- Processors fc and reporting - make_optimal_processors_featureclass(the_scenario, logger) - - # -- Ultimate Destinations fc and reporting - make_optimal_destinations_featureclass(the_scenario, logger) - - -# ====================================================================================================================== - - -def make_optimal_facilities_db(the_scenario, logger): - logger.info("starting make_optimal_facilities_db") - - # use the optimal solution and edges tables in the db to reconstruct what facilities are used - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_facilities" - db_con.execute(sql) - - # create the table - sql = """ - create table optimal_facilities( - facility_id integer, - facility_name text, - facility_type text, - facility_type_id integer, - time_period text, - commodity text - );""" - - db_con.execute(sql) - - sql = """ - insert into optimal_facilities - select f.facility_ID, f.facility_name, fti.facility_type, f.facility_type_id, ov.time_period, ov.commodity_name - from facilities f - join facility_type_id fti on f.facility_type_id = fti.facility_type_id - join optimal_variables ov on ov.o_facility = f.facility_name - join optimal_variables ov2 on ov2.d_facility = f.facility_name - where ov.o_facility is not NULL and ov2.d_facility is not NULL - ;""" - - db_con.execute(sql) - - -# ====================================================================================================================== - - -def make_optimal_intermodal_db(the_scenario, logger): - logger.info("starting make_optimal_intermodal_db") - - # use the optimal solution and edges tables in the db to reconstruct what facilities are used - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_intermodal_facilities" - db_con.execute(sql) - - # create the table - sql = """ - create table optimal_intermodal_facilities( - source text, - source_OID integer - );""" - - db_con.execute(sql) - - sql = """ - insert into optimal_intermodal_facilities - select nx_n.source, nx_n.source_OID - from networkx_nodes nx_n - join optimal_variables ov on ov.from_node_id = nx_n.node_id - join optimal_variables ov2 on ov2.to_node_id = nx_n.node_id - where nx_n.source = 'intermodal' - group by nx_n.source_OID - ;""" - - db_con.execute(sql) - - -# ====================================================================================================================== - - -def make_optimal_intermodal_from_routes_db(the_scenario, logger): - logger.info("starting make_optimal_intermodal_from_routes_db") - - # use the optimal solution and edges tables in the db to reconstruct what facilities are used - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_intermodal_facilities" - db_con.execute(sql) - - # create the table - sql = """ - create table optimal_intermodal_facilities( - source text, - source_OID integer - );""" - - db_con.execute(sql) - - sql = """ - insert into optimal_intermodal_facilities - select nx_n.source, nx_n.source_OID - from networkx_nodes nx_n - join optimal_route_segments ors1 on ors1.from_node_id = nx_n.node_id - join optimal_route_segments ors2 on ors2.to_node_id = nx_n.node_id - where nx_n.source = 'intermodal' - group by nx_n.source_OID - ;""" - - db_con.execute(sql) - - -# ====================================================================================================================== - - - -def make_optimal_intermodal_featureclass(the_scenario, logger): - - logger.info("starting make_optimal_intermodal_featureclass") - - intermodal_fc = os.path.join(the_scenario.main_gdb, "network", "intermodal") - - for field in arcpy.ListFields(intermodal_fc): - - if field.name.lower() =="optimal": - arcpy.DeleteField_management(intermodal_fc , "optimal") - - arcpy.AddField_management(intermodal_fc, "optimal", "DOUBLE") - - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - # get a list of the optimal raw_material_producer facilities - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select source_oid from optimal_intermodal_facilities;""" - db_cur = db_con.execute(sql) - intermodal_db_data = db_cur.fetchall() - - # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. - with arcpy.da.UpdateCursor(intermodal_fc, ["source_OID", "optimal"]) as cursor: - for row in cursor: - source_OID = row[0] - row[1] = 0 # assume to start, it is not an optimal facility - for opt_fac in intermodal_db_data: - if source_OID in opt_fac: # if the OID matches and optimal facility - row[1] = 1 # give it a positive value since we're not keep track of flows. - cursor.updateRow(row) - - edit.stopOperation() - edit.stopEditing(True) - - -# ====================================================================================================================== - - -def make_optimal_raw_material_producer_featureclass(the_scenario, logger): - - logger.info("starting make_optimal_raw_material_producer_featureclass") - # add rmp flows to rmp fc - # ---------------------------------------------------- - - rmp_fc = the_scenario.rmp_fc - - for field in arcpy.ListFields(rmp_fc ): - - if field.name.lower() == "optimal": - arcpy.DeleteField_management(rmp_fc, "optimal") - - arcpy.AddField_management(rmp_fc, "optimal", "DOUBLE") - - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - # get a list of the optimal raw_material_producer facilities - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select facility_name from optimal_facilities where facility_type = "raw_material_producer";""" - db_cur = db_con.execute(sql) - rmp_db_data = db_cur.fetchall() - - # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. - with arcpy.da.UpdateCursor(rmp_fc, ["facility_name", "optimal"]) as cursor: - for row in cursor: - facility_name = row[0] - row[1] = 0 # assume to start, it is not an optimal facility - for opt_fac in rmp_db_data: - - if facility_name in opt_fac: # if the name matches and optimal facility - - row[1] = 1 # give it a positive value since we're not keep track of flows. - cursor.updateRow(row) - - edit.stopOperation() - edit.stopEditing(True) - - -# ====================================================================================================================== - - -def make_optimal_processors_featureclass(the_scenario, logger): - logger.info("starting make_optimal_processors_featureclass") - - # query the db and get a list of optimal processors from the optimal_facilities table in the db. - # iterate through the processors_FC and see if the name of the facility match - # set the optimal field in the GIS if there is a match - - processor_fc = the_scenario.processors_fc - - for field in arcpy.ListFields(processor_fc): - - if field.name.lower() == "optimal": - arcpy.DeleteField_management(processor_fc, "optimal") - - arcpy.AddField_management(processor_fc, "optimal", "DOUBLE") - - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - # get a list of the optimal raw_material_producer facilities - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select facility_name from optimal_facilities where facility_type = "processor";""" - db_cur = db_con.execute(sql) - opt_fac_db_data = db_cur.fetchall() - - # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. - with arcpy.da.UpdateCursor(processor_fc, ["facility_name", "optimal"]) as cursor: - for row in cursor: - facility_name = row[0] - row[1] = 0 # assume to start, it is not an optimal facility - for opt_fac in opt_fac_db_data: - if facility_name in opt_fac: # if the name matches and optimal facility - row[1] = 1 # give it a positive value since we're not keep track of flows. - cursor.updateRow(row) - - edit.stopOperation() - edit.stopEditing(True) - - -# =================================================================================================== - - -def make_optimal_destinations_featureclass(the_scenario, logger): - logger.info("starting make_optimal_destinations_featureclass") - - # query the db and get a list of optimal processors from the optimal_facilities table in the db. - # iterate through the processors_FC and see if the name of the facility matches - # set the optimal field in the GIS if there is a match - - destinations_fc = the_scenario.destinations_fc - - for field in arcpy.ListFields(destinations_fc): - - if field.name.lower() == "optimal": - arcpy.DeleteField_management(destinations_fc , "optimal") - - arcpy.AddField_management(destinations_fc, "optimal", "DOUBLE") - - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - # get a list of the optimal raw_material_producer facilities - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select facility_name from optimal_facilities where facility_type = "ultimate_destination";""" - db_cur = db_con.execute(sql) - opt_fac_db_data = db_cur.fetchall() - - # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. - with arcpy.da.UpdateCursor(destinations_fc, ["facility_name", "optimal"]) as cursor: - for row in cursor: - facility_name = row[0] - row[1] = 0 # assume to start, it is not an optimal facility - for opt_fac in opt_fac_db_data: - if facility_name in opt_fac: # if the name matches and optimal facility - row[1] = 1 # give it a positive value since we're not keep track of flows. - cursor.updateRow(row) - - edit.stopOperation() - edit.stopEditing(True) - - scenario_gdb = the_scenario.main_gdb - - -# ===================================================================================================================== - - -def make_optimal_route_segments_db(the_scenario, logger): - # iterate through the db to create a dictionary of dictionaries (DOD) - # then generate the graph using the method - # >>> dod = {0: {1: {'weight': 1}}} # single edge (0,1) - # >>> G = nx.from_dict_of_dicts(dod) - # I believe this is read as the outer dict is the from_node, the inner dictionary - # is the to_node, the inner-inner dictionary is a list of attributes. - - logger.info("START: make_optimal_route_segments_db") - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_route_segments" - db_con.execute(sql) - - # create the table - sql = """ - create table optimal_route_segments( - scenario_rt_id integer, rt_variant_id integer, - network_source_id integer, network_source_oid integer, - from_position integer, from_junction_id integer, - time_period integer, - commodity_name text, commodity_flow real, - volume real, capacity real, capacity_minus_volume, - units text, phase_of_matter text, - miles real, route_type text, link_dollar_cost real, link_routing_cost real, - artificial integer, - FCLASS integer, - urban_rural_code integer - );""" - db_con.execute(sql) - - # intiialize dod - optimal_segments_list = [] - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_con.execute("""create index if not exists nx_edge_index_2 on networkx_edges(edge_id);""") - db_con.execute("""create index if not exists nx_edge_cost_index on networkx_edge_costs(edge_id);""") - db_con.execute("""create index if not exists ov_index on optimal_variables(nx_edge_id);""") - db_con.execute("""create index if not exists ov_index_2 on optimal_variables(commodity_name);""") - - sql = """ - select - ov.mode, - ov.mode_oid, - ov.commodity_name, - ov.variable_value, - ov.converted_volume, - ov.converted_capacity, - ov.converted_capac_minus_volume, - ov.units, - ov.time_period, - ov.miles, - c.phase_of_matter, - nx_e_cost.dollar_cost, - nx_e_cost.route_cost, - nx_e.artificial - from optimal_variables ov - join commodities as c on c.commodity_name = ov.commodity_name - join networkx_edges as nx_e on nx_e.edge_id = ov.nx_edge_id - join networkx_edge_costs as nx_e_cost on nx_e_cost.edge_id = ov.nx_edge_id and nx_e_cost.phase_of_matter_id = c.phase_of_matter - ;""" - - db_cur = db_con.execute(sql) - logger.info("done with the execute...") - - logger.info("starting the fetch all ....possible to increase speed with an index") - rows = db_cur.fetchall() - - row_count = len(rows) - logger.info("number of rows in the optimal segments select to process: {}".format(row_count)) - - logger.info("starting to iterate through the results and build up the dod") - for row in rows: - mode = row[0] - mode_oid = row[1] - commodity = row[2] - opt_flow = row[3] - volume = row[4] - capacity = row[5] - capacity_minus_volume = row[6] - units = row[7] - time_period = row[8] - miles = row[9] - phase_of_matter = row[10] - link_dollar_cost = row[11] - link_routing_cost = row[12] - artificial = row[13] - - # format for the optimal route segments table - optimal_segments_list.append([1, # rt id - None, # rt variant id - mode, - mode_oid, - None, # segment order - None, - time_period, - commodity, - opt_flow, - volume, - capacity, - capacity_minus_volume, - units, - phase_of_matter, - miles, - None, # route type - link_dollar_cost, - link_routing_cost, - artificial, - None, - None]) - - logger.info("done making the optimal_segments_list") - with sqlite3.connect(the_scenario.main_db) as db_con: - insert_sql = """ - INSERT into optimal_route_segments - values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - ;""" - - db_con.executemany(insert_sql, optimal_segments_list) - logger.info("finish optimal_segments_list db_con.executemany()") - db_con.commit() - logger.info("finish optimal_segments_list db_con.commit") - - return - - -# =================================================================================================== - - -def make_optimal_route_segments_from_routes_db(the_scenario, logger): - # iterate through the db to create a dictionary of dictionaries (DOD) - # then generate the graph using the method - # >>> dod = {0: {1: {'weight': 1}}} # single edge (0,1) - # >>> G = nx.from_dict_of_dicts(dod) - # I believe this is read as the outer dict is the from_node, the inner dictionary - # is the to_node, the inner-inner dictionary is a list of attributes. - - logger.info("START: make_optimal_route_segments_from_routes_db") - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_route_segments" - db_con.execute(sql) - - # create the table - db_con.execute("""create table optimal_route_segments( - scenario_rt_id integer, rt_variant_id integer, - network_source_id integer, network_source_oid integer, - from_position integer, from_junction_id integer, - time_period integer, - commodity_name text, commodity_flow real, - volume real, capacity real, capacity_minus_volume, - units text, phase_of_matter text, - miles real, route_type text, link_dollar_cost real, link_routing_cost real, - artificial integer, - FCLASS integer, - urban_rural_code integer, from_node_id integer, to_node_id integer - );""") - - db_con.execute("""create index if not exists nx_edge_index_2 on networkx_edges(edge_id);""") - db_con.execute("""create index if not exists nx_edge_cost_index on networkx_edge_costs(edge_id);""") - db_con.execute("""create index if not exists ov_index on optimal_variables(nx_edge_id);""") - db_con.execute("""create index if not exists ov_index_2 on optimal_variables(commodity_name);""") - - sql = """ - select - nx_e.mode_source, - nx_e.mode_source_oid, - ov.commodity_name, - ov.variable_value, - ov.converted_volume, - ov.converted_capacity, - ov.converted_capac_minus_volume, - ov.units, - ov.time_period, - nx_e.miles, - c.phase_of_matter, - nx_e_cost.dollar_cost, - nx_e_cost.route_cost , - nx_e.artificial, - re.scenario_rt_id, - nx_e.from_node_id, - nx_e.to_node_id - from optimal_variables ov - join commodities as c on c.commodity_name = ov.commodity_name - join edges as e on e.edge_id = ov.var_id - join route_reference as rr on rr.route_id = e.route_id - join route_edges as re on re.scenario_rt_id = rr.scenario_rt_id - left join networkx_edges as nx_e on nx_e.edge_id = re.edge_id - join networkx_edge_costs as nx_e_cost on nx_e_cost.edge_id = re.edge_id and nx_e_cost.phase_of_matter_id = c.phase_of_matter - ;""" - - db_cur = db_con.execute(sql) - logger.info("done with the execute...") - - logger.info("starting the fetch all ....possible to increase speed with an index") - rows = db_cur.fetchall() - - row_count = len(rows) - logger.info("number of rows in the optimal segments select to process: {}".format(row_count)) - - # intiialize dod - optimal_segments_list = [] - logger.info("starting to iterate through the results and build up the dod") - for row in rows: - mode = row[0] - mode_oid = row[1] - commodity = row[2] - opt_flow = row[3] - volume = row[4] - capacity = row[5] - capacity_minus_volume = row[6] - units = row[7] - time_period = row[8] - miles = row[9] - phase_of_matter = row[10] - link_dollar_cost = row[11] - link_routing_cost = row[12] - artificial = row[13] - scenario_rt_id = row[14] - from_node_id = row[15] - to_node_id = row[16] - - # format for the optimal route segments table - optimal_segments_list.append([scenario_rt_id, # rt id - None, # rt variant id - mode, - mode_oid, - None, # segment order - None, - time_period, - commodity, - opt_flow, - volume, - capacity, - capacity_minus_volume, - units, - phase_of_matter, - miles, - None, # route type - link_dollar_cost, - link_routing_cost, - artificial, - None, - None, - from_node_id, - to_node_id]) - - logger.info("done making the optimal_segments_list") - with sqlite3.connect(the_scenario.main_db) as db_con: - insert_sql = """ - INSERT into optimal_route_segments - values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - ;""" - - db_con.executemany(insert_sql, optimal_segments_list) - logger.info("finish optimal_segments_list db_con.executemany()") - db_con.commit() - logger.info("finish optimal_segments_list db_con.commit") - - return - - -# =================================================================================================== - - -def make_optimal_scenario_results_db(the_scenario, logger): - logger.info("starting make_optimal_scenario_results_db") - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists optimal_scenario_results" - db_con.execute(sql) - - # create the table - sql = """ - create table optimal_scenario_results( - table_name text, - commodity text, - facility_name text, - measure text, - mode text, - value real, - units text, - notes text - );""" - db_con.execute(sql) - - # sum all the flows on artificial = 1, and divide by 2 for each commodity. - # this assumes we have flows leaving and entering a facility on 1 artificial link at the beginning and the end. - sql_total_flow = """ -- total flow query - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'total_flow', - network_source_id, - sum(commodity_flow)/2, - units, - "" - from optimal_route_segments - where artificial = 1 - group by commodity_name, network_source_id - ;""" - db_con.execute(sql_total_flow) - - # miles by mode - # total network miles, not route miles - sql_total_miles = """ -- total scenario miles (not route miles) - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'miles', - network_source_id, - sum(miles), - 'miles', - '' - from - (select commodity_name, network_source_id, min(miles) as miles,network_source_oid from - optimal_route_segments group by network_source_oid, commodity_name, network_source_id) - group by commodity_name, network_source_id - ;""" - db_con.execute(sql_total_miles) - - # liquid unit-miles - sql_liquid_unit_miles = """ -- total liquid unit-miles by mode - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - "{}-miles", - network_source_id, - sum(commodity_flow*miles), - '{}-miles', - "" - from optimal_route_segments - where units = '{}' - group by commodity_name, network_source_id - ;""".format(the_scenario.default_units_liquid_phase, the_scenario.default_units_liquid_phase, the_scenario.default_units_liquid_phase) - db_con.execute(sql_liquid_unit_miles) - - # solid unit-miles for completeness - sql_solid_unit_miles = """ -- total solid unit-miles by mode - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - "{}-miles", - network_source_id, - sum(commodity_flow*miles), - '{}-miles', - "" - from optimal_route_segments - where units = '{}' - group by commodity_name, network_source_id - ;""".format(the_scenario.default_units_solid_phase, the_scenario.default_units_solid_phase, the_scenario.default_units_solid_phase) - db_con.execute(sql_solid_unit_miles) - - # dollar cost and routing cost by mode - # multiply the mileage by the flow - sql_dollar_costs = """ -- total dollar_cost - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - "dollar_cost", - network_source_id, - sum(commodity_flow*link_dollar_cost), - 'USD', - "" - from optimal_route_segments - group by commodity_name, network_source_id - ;""" - db_con.execute(sql_dollar_costs) - - sql_routing_costs = """ -- total routing_cost - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - "routing_cost", - network_source_id, - sum(commodity_flow*link_routing_cost), - 'USD', - "" - from optimal_route_segments - group by commodity_name, network_source_id - ;""" - db_con.execute(sql_routing_costs) - - # loads by mode and commodity - # use artificial links =1 and = 2 to calculate loads per loading event - # (e.g. leaving a facility or switching modes at intermodal facility) - - sql_mode_commodity = "select network_source_id, commodity_name from optimal_route_segments group by network_source_id, commodity_name;" - db_cur = db_con.execute(sql_mode_commodity) - mode_and_commodity_list = db_cur.fetchall() - - attributes_dict = ftot_supporting_gis.get_commodity_vehicle_attributes_dict(the_scenario, logger) - - for row in mode_and_commodity_list: - mode = row[0] - if 'pipeline_crude' in mode: - measure_name = "pipeline_crude_mvts" - if 'pipeline_prod' in mode: - measure_name = "pipeline_prod_mvts" - if "road" in mode: - measure_name = "truck_loads" - if "rail" in mode: - measure_name = "rail_cars" - if "water" in mode: - measure_name = "barge_loads" - commodity_name = row[1] - vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude - sql_vehicle_load = """ -- total loads by mode and commodity - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'vehicles', - network_source_id, - round(sum(commodity_flow/{}/2)), - '{}', -- units - '' - from optimal_route_segments - where (artificial = 1 or artificial = 2) and network_source_id = '{}' and commodity_name = '{}' - group by commodity_name, network_source_id - ;""".format(vehicle_payload, measure_name, mode, commodity_name) - db_con.execute(sql_vehicle_load) - - # VMT - # Do not report for pipeline - - for row in mode_and_commodity_list: - mode = row[0] - commodity_name = row[1] - - if 'pipeline' in mode: - pass - else: - vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude - # vmt is loads multiplied by miles - # we know the flow on a link we can calculate the loads on that link - # and multiply by the miles to get VMT. - sql_vmt = """ -- VMT by mode and commodity - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'vmt', - '{}', - sum(commodity_flow * miles/{}), - 'VMT', - "" - from optimal_route_segments - where network_source_id = '{}' and commodity_name = '{}' - group by commodity_name, network_source_id - ;""".format(mode, vehicle_payload, mode, commodity_name) - db_con.execute(sql_vmt) - - # CO2-- convert commodity and miles to CO2 - # trucks is a bit different than rail, and water - # fclass = 1 for interstate - # rural_urban_code == 99999 for rural, all else urban - # road CO2 emission factors in g/mi - - fclass_and_urban_code = {} - fclass_and_urban_code["CO2urbanRestricted"] = {'where': "urban_rural_code < 99999 and fclass = 1"} - fclass_and_urban_code["CO2urbanUnrestricted"] = {'where': "urban_rural_code < 99999 and fclass <> 1"} - fclass_and_urban_code["CO2ruralRestricted"] = {'where': "urban_rural_code = 99999 and fclass = 1"} - fclass_and_urban_code["CO2ruralUnrestricted"] = {'where': "urban_rural_code = 99999 and fclass <> 1"} - - for row in mode_and_commodity_list: - mode = row[0] - commodity_name = row[1] - if 'road' == mode: - vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude - for road_co2_measure_name in fclass_and_urban_code: - co2_val = attributes_dict[commodity_name][mode][road_co2_measure_name].magnitude #g/mi - where_clause = fclass_and_urban_code[road_co2_measure_name]['where'] - sql_road_co2 = """ -- CO2 for road by fclass and urban_rural_code - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - '{}', -- measure_name - '{}', -- mode - sum(commodity_flow * miles * {} /{}), -- value (VMT * co2 scaler) - 'grams', -- units - '' - from optimal_route_segments - where network_source_id = '{}' and commodity_name = '{}' and {} -- additional where_clause - group by commodity_name, network_source_id - ;""".format(road_co2_measure_name, mode, co2_val, vehicle_payload, mode, commodity_name, where_clause) - db_con.execute(sql_road_co2) - - # now total the road co2 values - sql_co2_road_total = """insert into optimal_scenario_results - select - 'commodity_summary', - commodity, - NULL, - 'co2', - mode, - sum(osr.value), - units, - "" - from optimal_scenario_results osr - where osr.measure like 'CO2%ed' - group by commodity - ;""" - db_con.execute(sql_co2_road_total) - - # now delete the intermediate records - sql_co2_delete = """delete - from optimal_scenario_results - where measure like 'CO2%ed' - ;""" - db_con.execute(sql_co2_delete) - - else: - co2_emissions = attributes_dict[commodity_name][mode]['CO2_Emissions'].magnitude #g/ton/mi - sql_co2 = """ -- CO2 by mode and commodity - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'co2', - '{}', - sum(commodity_flow * miles * {}), - 'grams', - "" - from optimal_route_segments - where network_source_id = '{}' and commodity_name = '{}' - group by commodity_name, network_source_id - ;""".format(mode, co2_emissions, mode, commodity_name) - db_con.execute(sql_co2) - - # Fuel burn - # covert VMT to fuel burn - # truck, rail, barge. no pipeline - # Same as VMT but divide result by fuel efficiency - - for row in mode_and_commodity_list: - mode = row[0] - commodity_name = row[1] - if 'pipeline' in mode: - pass - else: - vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude - fuel_efficiency = attributes_dict[commodity_name][mode]['Fuel_Efficiency'].magnitude #mi/gal - - # vmt is loads multiplied by miles - # we know the flow on a link we can calculate the loads on that link - # and multiply by the miles to get VMT. Then divide by fuel efficiency to get fuel burn - sql_fuel_burn = """ -- Fuel burn by mode and commodity - insert into optimal_scenario_results - select - 'commodity_summary', - commodity_name, - NULL, - 'fuel_burn', - '{}', - sum(commodity_flow * miles/{}/{}), - 'Gallons', - '' - from optimal_route_segments - where network_source_id = '{}' and commodity_name = '{}' - group by commodity_name, network_source_id - ;""".format(mode, vehicle_payload, fuel_efficiency, mode, commodity_name) - db_con.execute(sql_fuel_burn) - - # Destination Deliveries - # look at artificial = 1 and join on facility_name or d_location - # compare optimal flow at the facility (flow on all links) - # to the destination demand in the facility commodities table. - # report % fulfillment by commodity - # Destination report - - sql_destination_demand = """insert into optimal_scenario_results - select - "facility_summary", - c.commodity_name, - f.facility_name, - "destination_demand_total", - "total", - fc.quantity, - fc.units, - '' - from facility_commodities fc - join facilities f on f.facility_id = fc.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where fti.facility_type = 'ultimate_destination' - order by facility_name - ;""" - - # RMP supplier report - sql_rmp_supply = """insert into optimal_scenario_results - select - "facility_summary", - c.commodity_name, - f.facility_name, - "rmp_supply_total", - "total", - fc.quantity, - fc.units, - '' - from facility_commodities fc - join facilities f on f.facility_id = fc.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where fti.facility_type = 'raw_material_producer' - order by facility_name - ;""" - - # initialize SQL queries that vary based on whether routes are used or not - if the_scenario.ndrOn: - sql_destination_demand_optimal = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "destination_demand_optimal", - ne.mode_source, - sum(ov.variable_value), - ov.units, - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'ultimate_destination' and fc.commodity_id = ov.commodity_id - group by ov.d_facility, ov.commodity_name, ne.mode_source - ;""" - - sql_destination_demand_optimal_frac = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "destination_demand_optimal_frac", - ne.mode_source, - (sum(ov.variable_value) / fc.quantity), - "fraction", - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where ov.d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'ultimate_destination' and fc.commodity_id = ov.commodity_id - group by ov.d_facility, ov.commodity_name, ne.mode_source - ;""" - - # RMP supplier report - sql_rmp_supply_optimal = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "rmp_supply_optimal", - ne.mode_source, - sum(ov.variable_value), - ov.units, - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'raw_material_producer' and fc.commodity_id = ov.commodity_id - group by ov.o_facility, ov.commodity_name, ne.mode_source - ;""" - - # RMP supplier report - sql_rmp_supply_optimal_frac = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "rmp_supply_optimal_frac", - ne.mode_source, - --sum(ov.variable_value) as optimal_flow, - --fc.quantity as available_supply, - (sum(ov.variable_value) / fc.quantity), - "fraction", - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'raw_material_producer' and fc.commodity_id = ov.commodity_id - group by ov.o_facility, ov.commodity_name, ne.mode_source - ;""" - # Processor report - sql_processor_output = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "processor_output", - ne.mode_source, - (sum(ov.variable_value)), - ov.units, - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id - group by ov.o_facility, ov.commodity_name, ne.mode_source - ;""" - - sql_processor_input = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "processor_input", - ne.mode_source, - (sum(ov.variable_value)), - ov.units, - '' - from optimal_variables ov - join edges e on e.edge_id = ov.var_id - join route_reference rr on rr.route_id = e.route_id - join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where ov.d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id - group by ov.d_facility, ov.commodity_name, ne.mode_source - ;""" - else : - sql_destination_demand_optimal = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "destination_demand_optimal", - mode, - sum(ov.variable_value), - ov.units, - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'ultimate_destination' - group by d_facility, mode - ;""" - - sql_destination_demand_optimal_frac = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "destination_demand_optimal_frac", - mode, - (sum(ov.variable_value) / fc.quantity), - "fraction", - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'ultimate_destination' - group by d_facility, mode - ;""" - - # RMP optimal supply - sql_rmp_supply_optimal = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "rmp_supply_optimal", - mode, - sum(ov.variable_value), - ov.units, - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'raw_material_producer' - group by o_facility, mode - ;""" - - # RMP supplier report - sql_rmp_supply_optimal_frac = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "rmp_supply_optimal_frac", - mode, - --sum(ov.variable_value) as optimal_flow, - --fc.quantity as available_supply, - (sum(ov.variable_value) / fc.quantity), - "fraction", - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'raw_material_producer' - group by o_facility, mode - ;""" - - # Processor report - sql_processor_output = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.o_facility, - "processor_output", - mode, - (sum(ov.variable_value)), - ov.units, - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.o_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id - group by o_facility, mode, ov.commodity_name - ;""" - - sql_processor_input = """insert into optimal_scenario_results - select - "facility_summary", - ov.commodity_name, - ov.d_facility, - "processor_input", - mode, - (sum(ov.variable_value)), - ov.units, - '' - from optimal_variables ov - join facilities f on f.facility_name = ov.d_facility - join facility_commodities fc on fc.facility_id = f.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id - group by d_facility, mode, ov.commodity_name - ;""" - - db_con.execute(sql_destination_demand) - db_con.execute(sql_destination_demand_optimal) - db_con.execute(sql_destination_demand_optimal_frac) - db_con.execute(sql_rmp_supply) - db_con.execute(sql_rmp_supply_optimal) - db_con.execute(sql_rmp_supply_optimal_frac) - db_con.execute(sql_processor_output) - db_con.execute(sql_processor_input) - - # measure totals - sql_total = """insert into optimal_scenario_results - select table_name, commodity, facility_name, measure, "_total" as mode, sum(value), units, notes - from optimal_scenario_results - group by table_name, commodity, facility_name, measure - ;""" - db_con.execute(sql_total) - - logger.debug("finish: make_optimal_scenario_results_db()") - - -# =================================================================================================== - - -def generate_scenario_summary(the_scenario, logger): - logger.info("starting generate_scenario_summary") - - # query the optimal scenario results table and report out the results - with sqlite3.connect(the_scenario.main_db) as db_con: - - sql = "select * from optimal_scenario_results order by table_name, commodity, measure, mode;" - db_cur = db_con.execute(sql) - data = db_cur.fetchall() - - for row in data: - if row[2] == None: # these are the commodity summaries with no facility name - logger.result('{}_{}_{}_{}: \t {:,.2f} : \t {}'.format(row[0].upper(), row[3].upper(), row[1].upper(), row[4].upper(), row[5], row[6])) - else: # these are the commodity summaries with no facility name - logger.result('{}_{}_{}_{}_{}: \t {:,.2f} : \t {}'.format(row[0].upper(), row[2].upper(), row[3].upper(), row[1].upper(), row[4].upper(), row[5], row[6])) - - logger.debug("finish: generate_scenario_summary()") - - -# =================================================================================================== - - -def generate_artificial_link_summary(the_scenario, logger): - logger.info("starting generate_artificial_link_summary") - - # query the facility and network tables for artificial links and report out the results - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select fac.facility_name, fti.facility_type, ne.mode_source, round(ne.miles, 3) as miles - from facilities fac - left join facility_type_id fti on fac.facility_type_id = fti.facility_type_id - left join networkx_nodes nn on fac.location_id = nn.location_id - left join networkx_edges ne on nn.node_id = ne.from_node_id - where nn.location_1 like '%OUT%' - and ne.artificial = 1 - ;""" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - - artificial_links = {} - for row in db_data: - facility_name = row[0] - facility_type = row[1] - mode_source = row[2] - link_length = row[3] - - if facility_name not in artificial_links: - # add new facility to dictionary and start list of artificial links by mode - artificial_links[facility_name] = {'fac_type': facility_type, 'link_lengths': {}} - - if mode_source not in artificial_links[facility_name]['link_lengths']: - artificial_links[facility_name]['link_lengths'][mode_source] = link_length - else: - # there should only be one artificial link for a facility for each mode - error = "Multiple artificial links should not be found for a single facility for a particular mode." - logger.error(error) - raise Exception(error) - - # create structure for artificial link csv table - output_table = {'facility_name': [], 'facility_type': []} - for permitted_mode in the_scenario.permittedModes: - output_table[permitted_mode] = [] - - # iterate through every facility, add a row to csv table - for k in artificial_links: - output_table['facility_name'].append(k) - output_table['facility_type'].append(artificial_links[k]['fac_type']) - for permitted_mode in the_scenario.permittedModes: - if permitted_mode in artificial_links[k]['link_lengths']: - output_table[permitted_mode].append(artificial_links[k]['link_lengths'][permitted_mode]) - else: - output_table[permitted_mode].append('NA') - - # print artificial link data for each facility to file in debug folder - import csv - with open(os.path.join(the_scenario.scenario_run_directory, "debug", 'artificial_links.csv'), 'w', newline='') as f: - writer = csv.writer(f) - output_fields = ['facility_name', 'facility_type'] + the_scenario.permittedModes - writer.writerow(output_fields) - writer.writerows(zip(*[output_table[key] for key in output_fields])) - - logger.debug("finish: generate_artificial_link_summary()") - - -# ============================================================================================== - - -def db_report_commodity_utilization(the_scenario, logger): - logger.info("start: db_report_commodity_utilization") - - # This query pulls the total quantity flowed of each commodity from the optimal scenario results (osr) table. - # It groups by commodity_name, facility type, and units. The io field is included to help the user - # determine the potential supply, demand, and processing utilization in the scenario. - # ----------------------------------- - - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select c.commodity_name, - fti.facility_type, - io, - IFNULL((select osr.value - from optimal_scenario_results osr - where osr.commodity = c.commodity_name - and osr.measure ='total_flow' - and osr.mode = '_total'),-9999) optimal_flow, - c.units - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility == 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc - ;""" - db_cur = db_con.execute(sql) - - db_data = db_cur.fetchall() - logger.result("-------------------------------------------------------------------") - logger.result("Scenario Total Flow of Supply and Demand") - logger.result("-------------------------------------------------------------------") - logger.result("total utilization is defined as (total flow / net available)") - logger.result("commodity_name | facility_type | io | optimal_flow | units ") - logger.result("---------------|---------------|----|---------------|----------") - for row in db_data: - logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], - row[4])) - logger.result("-------------------------------------------------------------------") - - # This query compares the optimal flow to the net available quantity of material in the scenario. - # It groups by commodity_name, facility type, and units. The io field is included to help the user - # determine the potential supply, demand, and processing utilization in the scenario. - # ----------------------------------- - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """select c.commodity_name, - fti.facility_type, - io, - IFNULL(round((select osr.value - from optimal_scenario_results osr - where osr.commodity = c.commodity_name - and osr.measure ='total_flow' - and osr.mode = '_total') / sum(fc.quantity),2),-9999) Utilization, - 'fraction' - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility == 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc - ;""" - db_cur = db_con.execute(sql) - - db_data = db_cur.fetchall() - logger.result("-------------------------------------------------------------------") - logger.result("Scenario Total Utilization of Supply and Demand") - logger.result("-------------------------------------------------------------------") - logger.result("total utilization is defined as (total flow / net available)") - logger.result("commodity_name | facility_type | io | utilization | units ") - logger.result("---------------|---------------|----|---------------|----------") - for row in db_data: - logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], - row[4])) - logger.result("-------------------------------------------------------------------") - - -# =================================================================================================== - - -def make_optimal_route_segments_featureclass_from_db(the_scenario, - logger): - - logger.info("starting make_optimal_route_segments_featureclass_from_db") - - # create the segments layer - # ------------------------- - optimized_route_segments_fc = os.path.join(the_scenario.main_gdb, "optimized_route_segments") - - if arcpy.Exists(optimized_route_segments_fc): - arcpy.Delete_management(optimized_route_segments_fc) - logger.debug("deleted existing {} layer".format(optimized_route_segments_fc)) - - arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments", \ - "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ, "#", "0", "0", "0") - - arcpy.AddField_management(optimized_route_segments_fc, "ROUTE_TYPE", "TEXT", "#", "#", "25", "#", "NULLABLE", "NON_REQUIRED", "#") - arcpy.AddField_management(optimized_route_segments_fc, "FTOT_RT_ID", "LONG") - arcpy.AddField_management(optimized_route_segments_fc, "FTOT_RT_ID_VARIANT", "LONG") - arcpy.AddField_management(optimized_route_segments_fc, "NET_SOURCE_NAME", "TEXT") - arcpy.AddField_management(optimized_route_segments_fc, "NET_SOURCE_OID", "LONG") - arcpy.AddField_management(optimized_route_segments_fc, "ARTIFICIAL", "SHORT") - - # if from pos = 0 then traveresed in direction of underlying arc, otherwise traversed against the flow of the arc. - # used to determine if from_to_dollar cost should be used or to_from_dollar cost should be used. - arcpy.AddField_management(optimized_route_segments_fc, "FromPosition", "DOUBLE") # mnp 8/30/18 -- deprecated - arcpy.AddField_management(optimized_route_segments_fc, "FromJunctionID", "DOUBLE") # mnp - 060116 - added this for processor siting step - arcpy.AddField_management(optimized_route_segments_fc, "TIME_PERIOD", "TEXT") - arcpy.AddField_management(optimized_route_segments_fc, "COMMODITY", "TEXT") - arcpy.AddField_management(optimized_route_segments_fc, "COMMODITY_FLOW", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "VOLUME", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "CAPACITY", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "CAPACITY_MINUS_VOLUME", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "UNITS", "TEXT") - arcpy.AddField_management(optimized_route_segments_fc, "MILES", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "PHASE_OF_MATTER", "TEXT") - arcpy.AddField_management(optimized_route_segments_fc, "LINK_ROUTING_COST", "FLOAT") - arcpy.AddField_management(optimized_route_segments_fc, "LINK_DOLLAR_COST", "FLOAT") - - # get a list of the modes used in the optimal route_segments stored in the db. - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_cur = db_con.cursor() - - # get a list of the source_ids and convert to modes: - sql = "select distinct network_source_id from optimal_route_segments;" - db_cur = db_con.execute(sql) - rows = db_cur.fetchall() - - mode_source_list = [] - for row in rows: - mode_source_list.append(row[0]) - - logger.debug("List of modes used in the optimal solution: {}".format(mode_source_list)) - - # iterate through the segment_dict_by_source_name - # and get the segments to insert into the optimal segments layer - # --------------------------------------------------------------------- - - optimized_route_seg_flds = ("SHAPE@", "FTOT_RT_ID", "FTOT_RT_ID_VARIANT", "ROUTE_TYPE", "NET_SOURCE_NAME", - "NET_SOURCE_OID", "FromPosition", "FromJunctionID", "MILES", "TIME_PERIOD", "COMMODITY", "COMMODITY_FLOW", - "VOLUME", "CAPACITY", "CAPACITY_MINUS_VOLUME", "UNITS", "PHASE_OF_MATTER", "ARTIFICIAL", "LINK_ROUTING_COST", "LINK_DOLLAR_COST") - - for network_source in mode_source_list: - if network_source == 'intermodal': - logger.debug("network_source is: {}. can't flatten this, so skipping.".format(network_source)) - continue - - network_link_counter = 0 - network_link_coverage_counter = 0 - network_source_fc = os.path.join(the_scenario.main_gdb, "network", network_source) - - sql = """select DISTINCT - scenario_rt_id, - NULL, - network_source_id, - network_source_oid, - NULL, - NULL, - time_period, - commodity_name, - commodity_flow, - volume, - capacity, - capacity_minus_volume, - units, - phase_of_matter, - miles, - route_type, - artificial, - link_dollar_cost, - link_routing_cost - from optimal_route_segments - where network_source_id = '{}' - ;""".format(network_source) - - logger.debug("starting the execute for the {} mode".format(network_source)) - db_cur = db_con.execute(sql) - logger.debug("done with the execute") - logger.debug("starting fetch all for the {} mode".format(network_source)) - rows = db_cur.fetchall() - logger.debug("done with the fetchall") - - optimal_route_segments_dict = {} - - logger.debug("starting to build the dict for {}".format(network_source)) - for row in rows: - if row[3] not in optimal_route_segments_dict: - optimal_route_segments_dict[row[3]] = [] - network_link_coverage_counter += 1 - - route_id = row[0] - route_id_variant = row[1] - network_source_id = row[2] - network_object_id = row[3] - from_position = row[4] - from_junction_id = row[5] - time_period = row[6] - commodity = row[7] - commodity_flow = row[8] - volume = row[9] - capacity = row[10] - capacity_minus_volume= row[11] - units = row[12] - phase_of_matter = row[13] - miles = row[14] - route_type = row[15] - artificial = row[16] - link_dollar_cost = row[17] - link_routing_cost = row[18] - - optimal_route_segments_dict[row[3]].append([route_id, route_id_variant, network_source_id, - network_object_id, from_position, from_junction_id, time_period, - commodity, commodity_flow, volume, capacity, - capacity_minus_volume, units, phase_of_matter, miles, - route_type, artificial, link_dollar_cost, link_routing_cost]) - - logger.debug("done building the dict for {}".format(network_source)) - - logger.debug("starting the search cursor") - with arcpy.da.SearchCursor(network_source_fc, ["OBJECTID", "SHAPE@"]) as search_cursor: - logger.info("start: looping through the {} mode".format(network_source)) - for row in search_cursor: - network_link_counter += 1 - object_id = row[0] - geom = row[1] - - if object_id not in optimal_route_segments_dict: - continue - else: - for segment_info in optimal_route_segments_dict[object_id]: - (route_id, route_id_variant, network_source_id, network_object_id, from_position, - from_junction_id, time_period, commodity, commodity_flow, volume, capacity, capacity_minus_volume, - units, phase_of_matter, miles, route_type, artificial, link_dollar_cost, link_routing_cost) = segment_info - with arcpy.da.InsertCursor(optimized_route_segments_fc, optimized_route_seg_flds) as insert_cursor: - insert_cursor.insertRow([geom, route_id, route_id_variant, route_type, network_source_id, - network_object_id, from_position, from_junction_id, - miles, time_period, commodity, commodity_flow, volume, capacity, - capacity_minus_volume, units, phase_of_matter, artificial, - link_routing_cost, link_dollar_cost]) - - logger.debug("finish: looping through the {} mode".format(network_source)) - logger.info("mode: {} coverage: {:,.1f} - total links: {} , total links used: {}".format(network_source, - 100.0*(int(network_link_coverage_counter)/float(network_link_counter)), network_link_counter, network_link_coverage_counter )) - - -# ====================================================================================================================== - - -def add_fclass_and_urban_code(the_scenario, logger): - - logger.info("starting add_fclass_and_urban_code") - scenario_gdb = the_scenario.main_gdb - optimized_route_segments_fc = os.path.join(scenario_gdb, "optimized_route_segments") - - # build up dictionary network links we are interested in - # ---------------------------------------------------- - - network_source_oid_dict = {} - network_source_oid_dict['road'] = {} - - with arcpy.da.SearchCursor(optimized_route_segments_fc, ("NET_SOURCE_NAME", "NET_SOURCE_OID")) as search_cursor: - for row in search_cursor: - if row[0] == 'road': - network_source_oid_dict['road'][row[1]] = True - - # iterate different network layers (i.e. rail, road, water, pipeline) - # and get needed info into the network_source_oid_dict dict - # ----------------------------------------------------------- - - road_fc = os.path.join(scenario_gdb, "network", "road") - flds = ("OID@", "FCLASS", "URBAN_CODE") - with arcpy.da.SearchCursor(road_fc, flds) as search_cursor: - for row in search_cursor: - if row[0] in network_source_oid_dict['road']: - network_source_oid_dict['road'][row[0]] = [row[1], row[2]] - - # add fields to hold data from network links - # ------------------------------------------ - - # FROM ROAD NET ONLY - arcpy.AddField_management(optimized_route_segments_fc, "ROAD_FCLASS", "SHORT") - arcpy.AddField_management(optimized_route_segments_fc, "URBAN_CODE", "LONG") - - # set network link related values in optimized_route_segments layer - # ----------------------------------------------------------------- - - flds = ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ROAD_FCLASS", "URBAN_CODE"] - - # initialize a list for the update - - update_list = [] - - with arcpy.da.UpdateCursor(optimized_route_segments_fc, flds) as update_cursor: - - for row in update_cursor: - - net_source = row[0] - net_oid = row[1] - - if net_source == 'road': - data = network_source_oid_dict[net_source][net_oid] - row[2] = data[0] # fclass - row[3] = data[1] # urban_code - - update_cursor.updateRow(row) - - update_list.append([row[2], row[3], net_source, net_oid]) - - logger.debug("starting the execute many to update the list of optimal_route_segments") - with sqlite3.connect(the_scenario.main_db) as db_con: - update_sql = """ - UPDATE optimal_route_segments - set fclass = ?, urban_rural_code = ? - where network_source_id = ? and network_source_oid = ? - ;""" - - db_con.executemany(update_sql, update_list) - db_con.commit() - - -# ====================================================================================================================== - - -def dissolve_optimal_route_segments_feature_class_for_mapping(the_scenario, logger): - - # Make a dissolved version of fc for mapping aggregate flows - logger.info("starting dissolve_optimal_route_segments_feature_class_for_mapping") - - scenario_gdb = the_scenario.main_gdb - optimized_route_segments_fc = os.path.join(scenario_gdb, "optimized_route_segments") - - arcpy.env.workspace = scenario_gdb - - if arcpy.Exists("optimized_route_segments_dissolved"): - arcpy.Delete_management("optimized_route_segments_dissolved") - - arcpy.MakeFeatureLayer_management(optimized_route_segments_fc, "segments_lyr") - result = arcpy.GetCount_management("segments_lyr") - count = str(result.getOutput(0)) - - if arcpy.Exists("optimized_route_segments_dissolved_tmp"): - arcpy.Delete_management("optimized_route_segments_dissolved_tmp") - - if arcpy.Exists("optimized_route_segments_split_tmp"): - arcpy.Delete_management("optimized_route_segments_split_tmp") - - if arcpy.Exists("optimized_route_segments_dissolved_tmp2"): - arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") - - if int(count) > 0: - - # Dissolve - arcpy.Dissolve_management(optimized_route_segments_fc, "optimized_route_segments_dissolved_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", "PHASE_OF_MATTER", "UNITS"], - [['COMMODITY_FLOW', 'SUM']], "SINGLE_PART", "DISSOLVE_LINES") - - if arcpy.CheckProduct("ArcInfo") == "Available": - # Second dissolve needed to accurately show aggregate pipeline flows - arcpy.FeatureToLine_management("optimized_route_segments_dissolved_tmp", - "optimized_route_segments_split_tmp") - - arcpy.AddGeometryAttributes_management("optimized_route_segments_split_tmp", "LINE_START_MID_END") - - arcpy.Dissolve_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2", - ["NET_SOURCE_NAME", "Shape_Length", "MID_X", "MID_Y", "ARTIFICIAL", - "PHASE_OF_MATTER", "UNITS"], - [["SUM_COMMODITY_FLOW", "SUM"]], "SINGLE_PART", "DISSOLVE_LINES") - - arcpy.AddField_management("optimized_route_segments_dissolved_tmp2", "SUM_COMMODITY_FLOW", "DOUBLE") - arcpy.CalculateField_management("optimized_route_segments_dissolved_tmp2", "SUM_COMMODITY_FLOW", - "!SUM_SUM_COMMODITY_FLOW!", "PYTHON_9.3") - arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "SUM_SUM_COMMODITY_FLOW") - arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "MID_X") - arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "MID_Y") - - else: - # Doing it differently because feature to line isn't available without an advanced license - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. A modification to the " - "dissolve_optimal_route_segments_feature_class_for_mapping method is necessary") - - # Create the fc - arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments_split_tmp", - "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ) - - arcpy.AddField_management("optimized_route_segments_split_tmp", "NET_SOURCE_NAME", "TEXT") - arcpy.AddField_management("optimized_route_segments_split_tmp", "NET_SOURCE_OID", "LONG") - arcpy.AddField_management("optimized_route_segments_split_tmp", "ARTIFICIAL", "SHORT") - arcpy.AddField_management("optimized_route_segments_split_tmp", "UNITS", "TEXT") - arcpy.AddField_management("optimized_route_segments_split_tmp", "PHASE_OF_MATTER", "TEXT") - arcpy.AddField_management("optimized_route_segments_split_tmp", "SUM_COMMODITY_FLOW", "DOUBLE") - - # Go through the pipeline segments separately - tariff_segment_dict = defaultdict(float) - with arcpy.da.SearchCursor("optimized_route_segments_dissolved_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", "PHASE_OF_MATTER", "UNITS", - "SUM_COMMODITY_FLOW", "SHAPE@"]) as search_cursor: - for row1 in search_cursor: - if 'pipeline' in row1[0]: - # Must not be artificial, otherwise pass the link through - if row1[2] == 0: - # Capture the tariff ID so that we can link to the segments - mode = row1[0] - with arcpy.da.SearchCursor(mode, ["OBJECTID", "Tariff_ID", "SHAPE@"]) \ - as search_cursor_2: - for row2 in search_cursor_2: - if row1[1] == row2[0]: - tariff_id = row2[1] - mode = row1[0].strip("rts") - with arcpy.da.SearchCursor(mode + "sgmts", ["MASTER_OID", "Tariff_ID", "SHAPE@"]) \ - as search_cursor_3: - for row3 in search_cursor_3: - if tariff_id == row3[1]: - # keying off master_oid, net_source_name, phase of matter, units + shape - tariff_segment_dict[(row3[0], row1[0], row1[3], row1[4], row3[2])] += row1[5] - else: - with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", - "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"]) \ - as insert_cursor: - insert_cursor.insertRow([row1[0], row1[1], row1[2], row1[3], row1[4], row1[5], row1[6]]) - # If it isn't pipeline just pass the data through. - else: - with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", - "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"])\ - as insert_cursor: - insert_cursor.insertRow([row1[0], row1[1], row1[2], row1[3], row1[4], row1[5], row1[6]]) - - # Now that pipeline segment dictionary is built, get the pipeline segments in there as well - for master_oid, net_source_name, phase_of_matter, units, shape in tariff_segment_dict: - commodity_flow = tariff_segment_dict[master_oid, net_source_name, phase_of_matter, units, shape] - with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", - ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", - "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"]) \ - as insert_cursor: - insert_cursor.insertRow([net_source_name, master_oid, 0, phase_of_matter, units, commodity_flow, - shape]) - # No need for dissolve because dictionaries have already summed flows - arcpy.Copy_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2") - - # Sort for mapping order - arcpy.AddField_management("optimized_route_segments_dissolved_tmp2", "SORT_FIELD", "SHORT") - arcpy.MakeFeatureLayer_management("optimized_route_segments_dissolved_tmp2", "dissolved_segments_lyr") - arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'road'") - arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 1, "PYTHON_9.3") - arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'rail'") - arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 2, "PYTHON_9.3") - arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'water'") - arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 3, "PYTHON_9.3") - arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", - "NET_SOURCE_NAME LIKE 'pipeline%'") - arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 4, "PYTHON_9.3") - - arcpy.Sort_management("optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved", - [["SORT_FIELD", "ASCENDING"]]) - - # Delete temp fc's - arcpy.Delete_management("optimized_route_segments_dissolved_tmp") - arcpy.Delete_management("optimized_route_segments_split_tmp") - arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") - - else: - arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments_dissolved", - "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ, "#", - "0", "0", "0") - - arcpy.AddField_management("optimized_route_segments_dissolved", "NET_SOURCE_NAME", "TEXT") - arcpy.AddField_management("optimized_route_segments_dissolved", "ARTIFICIAL", "SHORT") - arcpy.AddField_management("optimized_route_segments_dissolved", "UNITS", "TEXT") - arcpy.AddField_management("optimized_route_segments_dissolved", "PHASE_OF_MATTER", "TEXT") - arcpy.AddField_management("optimized_route_segments_dissolved", "SUM_COMMODITY_FLOW", "DOUBLE") - - arcpy.Delete_management("segments_lyr") - +# --------------------------------------------------------------------------------------------------- +# Name: ftot_postprocess.py +# +# Purpose: Post processes the optimal results, generates optimal segments FC and +# optimal_scenario_results table in the database +# +# --------------------------------------------------------------------------------------------------- + +import ftot_supporting_gis +import arcpy +import sqlite3 +import os +from collections import defaultdict + + +# ========================================================================= + + +def route_post_optimization_db(the_scenario, logger): + + logger.info("starting route_post_optimization_db") + + # Parse the Optimal Solution from the DB + # --------------------------------------- + from ftot_pulp import parse_optimal_solution_db + parsed_optimal_solution = parse_optimal_solution_db(the_scenario, logger) + optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material = parsed_optimal_solution + + if not the_scenario.ndrOn: + # Make the optimal routes and route_segments FCs from the db + # ------------------------------------------------------------ + make_optimal_route_segments_db(the_scenario, logger) + elif the_scenario.ndrOn: + # Make the optimal routes and route_segments FCs from the db + # ------------------------------------------------------------ + make_optimal_route_segments_from_routes_db(the_scenario, logger) + + # Make the optimal routes and route segments FCs + # ----------------------------------------------- + # One record for each link in the optimal flow by commodity + make_optimal_route_segments_featureclass_from_db(the_scenario, logger) + + # Add functional class and urban code to the road segments + add_fclass_and_urban_code(the_scenario, logger) + + dissolve_optimal_route_segments_feature_class_for_mapping(the_scenario, logger) + + make_optimal_scenario_results_db(the_scenario, logger) + + # Report the utilization by commodity + # ------------------------------------ + db_report_commodity_utilization(the_scenario, logger) + + # Generate the scenario summary + # ------------------------------ + generate_scenario_summary(the_scenario, logger) + + # Calculate detailed emissions + detailed_emissions_setup(the_scenario, logger) + + # Make the optimal intermodal facilities + # ---------------------------- + if not the_scenario.ndrOn: + make_optimal_intermodal_db(the_scenario, logger) + elif the_scenario.ndrOn: + make_optimal_intermodal_from_routes_db(the_scenario, logger) + + make_optimal_intermodal_featureclass(the_scenario, logger) + + # Make the optimal facilities + # ---------------------------- + make_optimal_facilities_db(the_scenario, logger) + + # Now that we have a table with the optimal facilities, we can go through each FC and see if the name + # matches an "optimal" facility_name, and set the flag. + + # -- RMP fc and reporting + make_optimal_raw_material_producer_featureclass(the_scenario, logger) + + # -- Processors fc and reporting + make_optimal_processors_featureclass(the_scenario, logger) + + # -- Ultimate Destinations fc and reporting + make_optimal_destinations_featureclass(the_scenario, logger) + + +# ====================================================================================================================== + + +def make_optimal_facilities_db(the_scenario, logger): + logger.info("starting make_optimal_facilities_db") + + # use the optimal solution and edges tables in the db to reconstruct what facilities are used + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_facilities" + db_con.execute(sql) + + # create the table + sql = """ + create table optimal_facilities( + facility_id integer, + facility_name text, + facility_type text, + facility_type_id integer, + time_period text, + commodity text + );""" + + db_con.execute(sql) + + sql = """ + insert into optimal_facilities + select f.facility_ID, f.facility_name, fti.facility_type, f.facility_type_id, ov.time_period, ov.commodity_name + from facilities f + join facility_type_id fti on f.facility_type_id = fti.facility_type_id + join optimal_variables ov on ov.o_facility = f.facility_name + join optimal_variables ov2 on ov2.d_facility = f.facility_name + where ov.o_facility is not NULL and ov2.d_facility is not NULL + ;""" + + db_con.execute(sql) + + +# ====================================================================================================================== + + +def make_optimal_intermodal_db(the_scenario, logger): + logger.info("starting make_optimal_intermodal_db") + + # use the optimal solution and edges tables in the db to reconstruct what facilities are used + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_intermodal_facilities" + db_con.execute(sql) + + # create the table + sql = """ + create table optimal_intermodal_facilities( + source text, + source_OID integer + );""" + + db_con.execute(sql) + + sql = """ + insert into optimal_intermodal_facilities + select nx_n.source, nx_n.source_OID + from networkx_nodes nx_n + join optimal_variables ov on ov.from_node_id = nx_n.node_id + join optimal_variables ov2 on ov2.to_node_id = nx_n.node_id + where nx_n.source = 'intermodal' + group by nx_n.source_OID + ;""" + + db_con.execute(sql) + + +# ====================================================================================================================== + + +def make_optimal_intermodal_from_routes_db(the_scenario, logger): + logger.info("starting make_optimal_intermodal_from_routes_db") + + # use the optimal solution and edges tables in the db to reconstruct what facilities are used + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_intermodal_facilities" + db_con.execute(sql) + + # create the table + sql = """ + create table optimal_intermodal_facilities( + source text, + source_OID integer + );""" + + db_con.execute(sql) + + sql = """ + insert into optimal_intermodal_facilities + select nx_n.source, nx_n.source_OID + from networkx_nodes nx_n + join optimal_route_segments ors1 on ors1.from_node_id = nx_n.node_id + join optimal_route_segments ors2 on ors2.to_node_id = nx_n.node_id + where nx_n.source = 'intermodal' + group by nx_n.source_OID + ;""" + + db_con.execute(sql) + + +# ====================================================================================================================== + + + +def make_optimal_intermodal_featureclass(the_scenario, logger): + + logger.info("starting make_optimal_intermodal_featureclass") + + intermodal_fc = os.path.join(the_scenario.main_gdb, "network", "intermodal") + + for field in arcpy.ListFields(intermodal_fc): + + if field.name.lower() =="optimal": + arcpy.DeleteField_management(intermodal_fc , "optimal") + + arcpy.AddField_management(intermodal_fc, "optimal", "DOUBLE") + + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + # get a list of the optimal raw_material_producer facilities + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select source_oid from optimal_intermodal_facilities;""" + db_cur = db_con.execute(sql) + intermodal_db_data = db_cur.fetchall() + + # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. + with arcpy.da.UpdateCursor(intermodal_fc, ["source_OID", "optimal"]) as cursor: + for row in cursor: + source_OID = row[0] + row[1] = 0 # assume to start, it is not an optimal facility + for opt_fac in intermodal_db_data: + if source_OID in opt_fac: # if the OID matches and optimal facility + row[1] = 1 # give it a positive value since we're not keep track of flows. + cursor.updateRow(row) + + edit.stopOperation() + edit.stopEditing(True) + + +# ====================================================================================================================== + + +def make_optimal_raw_material_producer_featureclass(the_scenario, logger): + + logger.info("starting make_optimal_raw_material_producer_featureclass") + # add rmp flows to rmp fc + # ---------------------------------------------------- + + rmp_fc = the_scenario.rmp_fc + + for field in arcpy.ListFields(rmp_fc ): + + if field.name.lower() == "optimal": + arcpy.DeleteField_management(rmp_fc, "optimal") + + arcpy.AddField_management(rmp_fc, "optimal", "DOUBLE") + + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + # get a list of the optimal raw_material_producer facilities + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select facility_name from optimal_facilities where facility_type = "raw_material_producer";""" + db_cur = db_con.execute(sql) + rmp_db_data = db_cur.fetchall() + + # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. + with arcpy.da.UpdateCursor(rmp_fc, ["facility_name", "optimal"]) as cursor: + for row in cursor: + facility_name = row[0] + row[1] = 0 # assume to start, it is not an optimal facility + for opt_fac in rmp_db_data: + + if facility_name in opt_fac: # if the name matches and optimal facility + + row[1] = 1 # give it a positive value since we're not keep track of flows. + cursor.updateRow(row) + + edit.stopOperation() + edit.stopEditing(True) + + +# ====================================================================================================================== + + +def make_optimal_processors_featureclass(the_scenario, logger): + logger.info("starting make_optimal_processors_featureclass") + + # query the db and get a list of optimal processors from the optimal_facilities table in the db. + # iterate through the processors_FC and see if the name of the facility match + # set the optimal field in the GIS if there is a match + + processor_fc = the_scenario.processors_fc + + for field in arcpy.ListFields(processor_fc): + + if field.name.lower() == "optimal": + arcpy.DeleteField_management(processor_fc, "optimal") + + arcpy.AddField_management(processor_fc, "optimal", "DOUBLE") + + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + # get a list of the optimal raw_material_producer facilities + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select facility_name from optimal_facilities where facility_type = "processor";""" + db_cur = db_con.execute(sql) + opt_fac_db_data = db_cur.fetchall() + + # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. + with arcpy.da.UpdateCursor(processor_fc, ["facility_name", "optimal"]) as cursor: + for row in cursor: + facility_name = row[0] + row[1] = 0 # assume to start, it is not an optimal facility + for opt_fac in opt_fac_db_data: + if facility_name in opt_fac: # if the name matches and optimal facility + row[1] = 1 # give it a positive value since we're not keep track of flows. + cursor.updateRow(row) + + edit.stopOperation() + edit.stopEditing(True) + + +# =================================================================================================== + + +def make_optimal_destinations_featureclass(the_scenario, logger): + logger.info("starting make_optimal_destinations_featureclass") + + # query the db and get a list of optimal processors from the optimal_facilities table in the db. + # iterate through the processors_FC and see if the name of the facility matches + # set the optimal field in the GIS if there is a match + + destinations_fc = the_scenario.destinations_fc + + for field in arcpy.ListFields(destinations_fc): + + if field.name.lower() == "optimal": + arcpy.DeleteField_management(destinations_fc , "optimal") + + arcpy.AddField_management(destinations_fc, "optimal", "DOUBLE") + + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + # get a list of the optimal raw_material_producer facilities + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select facility_name from optimal_facilities where facility_type = "ultimate_destination";""" + db_cur = db_con.execute(sql) + opt_fac_db_data = db_cur.fetchall() + + # loop through the GIS feature class and see if any of the facility_names match the list of optimal facilities. + with arcpy.da.UpdateCursor(destinations_fc, ["facility_name", "optimal"]) as cursor: + for row in cursor: + facility_name = row[0] + row[1] = 0 # assume to start, it is not an optimal facility + for opt_fac in opt_fac_db_data: + if facility_name in opt_fac: # if the name matches and optimal facility + row[1] = 1 # give it a positive value since we're not keep track of flows. + cursor.updateRow(row) + + edit.stopOperation() + edit.stopEditing(True) + + scenario_gdb = the_scenario.main_gdb + + +# ===================================================================================================================== + + +def make_optimal_route_segments_db(the_scenario, logger): + # iterate through the db to create a dictionary of dictionaries (DOD) + # then generate the graph using the method + # >>> dod = {0: {1: {'weight': 1}}} # single edge (0,1) + # >>> G = nx.from_dict_of_dicts(dod) + # I believe this is read as the outer dict is the from_node, the inner dictionary + # is the to_node, the inner-inner dictionary is a list of attributes. + + logger.info("START: make_optimal_route_segments_db") + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_route_segments" + db_con.execute(sql) + + # create the table + sql = """ + create table optimal_route_segments( + scenario_rt_id integer, rt_variant_id integer, + network_source_id integer, network_source_oid integer, + from_position integer, from_junction_id integer, + time_period integer, + commodity_name text, commodity_flow real, + volume real, capacity real, capacity_minus_volume, + units text, phase_of_matter text, + miles real, route_type text, link_dollar_cost real, link_routing_cost real, + artificial integer, + FCLASS integer, + urban_rural_code integer + );""" + db_con.execute(sql) + + # intiialize dod + optimal_segments_list = [] + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_con.execute("""create index if not exists nx_edge_index_2 on networkx_edges(edge_id);""") + db_con.execute("""create index if not exists nx_edge_cost_index on networkx_edge_costs(edge_id);""") + db_con.execute("""create index if not exists ov_index on optimal_variables(nx_edge_id);""") + db_con.execute("""create index if not exists ov_index_2 on optimal_variables(commodity_name);""") + + sql = """ + select + ov.mode, + ov.mode_oid, + ov.commodity_name, + ov.variable_value, + ov.converted_volume, + ov.converted_capacity, + ov.converted_capac_minus_volume, + ov.units, + ov.time_period, + ov.miles, + c.phase_of_matter, + nx_e_cost.dollar_cost, + nx_e_cost.route_cost, + nx_e.artificial + from optimal_variables ov + join commodities as c on c.commodity_name = ov.commodity_name + join networkx_edges as nx_e on nx_e.edge_id = ov.nx_edge_id + join networkx_edge_costs as nx_e_cost on nx_e_cost.edge_id = ov.nx_edge_id and nx_e_cost.phase_of_matter_id = c.phase_of_matter + ;""" + + db_cur = db_con.execute(sql) + logger.info("done with the execute...") + + logger.info("starting the fetch all ....possible to increase speed with an index") + rows = db_cur.fetchall() + + row_count = len(rows) + logger.info("number of rows in the optimal segments select to process: {}".format(row_count)) + + logger.info("starting to iterate through the results and build up the dod") + for row in rows: + mode = row[0] + mode_oid = row[1] + commodity = row[2] + opt_flow = row[3] + volume = row[4] + capacity = row[5] + capacity_minus_volume = row[6] + units = row[7] + time_period = row[8] + miles = row[9] + phase_of_matter = row[10] + link_dollar_cost = row[11] + link_routing_cost = row[12] + artificial = row[13] + + # format for the optimal route segments table + optimal_segments_list.append([1, # rt id + None, # rt variant id + mode, + mode_oid, + None, # segment order + None, + time_period, + commodity, + opt_flow, + volume, + capacity, + capacity_minus_volume, + units, + phase_of_matter, + miles, + None, # route type + link_dollar_cost, + link_routing_cost, + artificial, + None, + None]) + + logger.info("done making the optimal_segments_list") + with sqlite3.connect(the_scenario.main_db) as db_con: + insert_sql = """ + INSERT into optimal_route_segments + values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ;""" + + db_con.executemany(insert_sql, optimal_segments_list) + logger.info("finish optimal_segments_list db_con.executemany()") + db_con.commit() + logger.info("finish optimal_segments_list db_con.commit") + + return + + +# =================================================================================================== + + +def make_optimal_route_segments_from_routes_db(the_scenario, logger): + # iterate through the db to create a dictionary of dictionaries (DOD) + # then generate the graph using the method + # >>> dod = {0: {1: {'weight': 1}}} # single edge (0,1) + # >>> G = nx.from_dict_of_dicts(dod) + # I believe this is read as the outer dict is the from_node, the inner dictionary + # is the to_node, the inner-inner dictionary is a list of attributes. + + logger.info("START: make_optimal_route_segments_from_routes_db") + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_route_segments" + db_con.execute(sql) + + # create the table + db_con.execute("""create table optimal_route_segments( + scenario_rt_id integer, rt_variant_id integer, + network_source_id integer, network_source_oid integer, + from_position integer, from_junction_id integer, + time_period integer, + commodity_name text, commodity_flow real, + volume real, capacity real, capacity_minus_volume, + units text, phase_of_matter text, + miles real, route_type text, link_dollar_cost real, link_routing_cost real, + artificial integer, + FCLASS integer, + urban_rural_code integer, from_node_id integer, to_node_id integer + );""") + + db_con.execute("""create index if not exists nx_edge_index_2 on networkx_edges(edge_id);""") + db_con.execute("""create index if not exists nx_edge_cost_index on networkx_edge_costs(edge_id);""") + db_con.execute("""create index if not exists ov_index on optimal_variables(nx_edge_id);""") + db_con.execute("""create index if not exists ov_index_2 on optimal_variables(commodity_name);""") + + sql = """ + select + nx_e.mode_source, + nx_e.mode_source_oid, + ov.commodity_name, + ov.variable_value, + ov.converted_volume, + ov.converted_capacity, + ov.converted_capac_minus_volume, + ov.units, + ov.time_period, + nx_e.miles, + c.phase_of_matter, + nx_e_cost.dollar_cost, + nx_e_cost.route_cost , + nx_e.artificial, + re.scenario_rt_id, + nx_e.from_node_id, + nx_e.to_node_id + from optimal_variables ov + join commodities as c on c.commodity_name = ov.commodity_name + join edges as e on e.edge_id = ov.var_id + join route_reference as rr on rr.route_id = e.route_id + join route_edges as re on re.scenario_rt_id = rr.scenario_rt_id + left join networkx_edges as nx_e on nx_e.edge_id = re.edge_id + join networkx_edge_costs as nx_e_cost on nx_e_cost.edge_id = re.edge_id and nx_e_cost.phase_of_matter_id = c.phase_of_matter + ;""" + + db_cur = db_con.execute(sql) + logger.info("done with the execute...") + + logger.info("starting the fetch all ....possible to increase speed with an index") + rows = db_cur.fetchall() + + row_count = len(rows) + logger.info("number of rows in the optimal segments select to process: {}".format(row_count)) + + # intiialize dod + optimal_segments_list = [] + logger.info("starting to iterate through the results and build up the dod") + for row in rows: + mode = row[0] + mode_oid = row[1] + commodity = row[2] + opt_flow = row[3] + volume = row[4] + capacity = row[5] + capacity_minus_volume = row[6] + units = row[7] + time_period = row[8] + miles = row[9] + phase_of_matter = row[10] + link_dollar_cost = row[11] + link_routing_cost = row[12] + artificial = row[13] + scenario_rt_id = row[14] + from_node_id = row[15] + to_node_id = row[16] + + # format for the optimal route segments table + optimal_segments_list.append([scenario_rt_id, # rt id + None, # rt variant id + mode, + mode_oid, + None, # segment order + None, + time_period, + commodity, + opt_flow, + volume, + capacity, + capacity_minus_volume, + units, + phase_of_matter, + miles, + None, # route type + link_dollar_cost, + link_routing_cost, + artificial, + None, + None, + from_node_id, + to_node_id]) + + logger.info("done making the optimal_segments_list") + with sqlite3.connect(the_scenario.main_db) as db_con: + insert_sql = """ + INSERT into optimal_route_segments + values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + ;""" + + db_con.executemany(insert_sql, optimal_segments_list) + logger.info("finish optimal_segments_list db_con.executemany()") + db_con.commit() + logger.info("finish optimal_segments_list db_con.commit") + + return + + +# =================================================================================================== + + +def make_optimal_scenario_results_db(the_scenario, logger): + logger.info("starting make_optimal_scenario_results_db") + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists optimal_scenario_results" + db_con.execute(sql) + + # create the table + sql = """ + create table optimal_scenario_results( + table_name text, + commodity text, + facility_name text, + measure text, + mode text, + value real, + units text, + notes text + );""" + db_con.execute(sql) + + # sum all the flows on artificial = 1, and divide by 2 for each commodity. + # this assumes we have flows leaving and entering a facility on 1 artificial link at the beginning and the end. + logger.debug("start: summarize total flow") + sql_total_flow = """ -- total flow query + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'total_flow', + network_source_id, + sum(commodity_flow)/2, + units, + "" + from optimal_route_segments + where artificial = 1 + group by commodity_name, network_source_id + ;""" + db_con.execute(sql_total_flow) + + # miles by mode + # total network miles, not route miles + logger.debug("start: summarize total miles") + sql_total_miles = """ -- total scenario miles (not route miles) + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'miles', + network_source_id, + sum(miles), + 'miles', + '' + from + (select commodity_name, network_source_id, min(miles) as miles,network_source_oid from + optimal_route_segments group by network_source_oid, commodity_name, network_source_id) + group by commodity_name, network_source_id + ;""" + db_con.execute(sql_total_miles) + + logger.debug("start: summarize unit miles") + # liquid unit-miles + sql_liquid_unit_miles = """ -- total liquid unit-miles by mode + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + "{}-miles", + network_source_id, + sum(commodity_flow*miles), + '{}-miles', + "" + from optimal_route_segments + where units = '{}' + group by commodity_name, network_source_id + ;""".format(the_scenario.default_units_liquid_phase, the_scenario.default_units_liquid_phase, the_scenario.default_units_liquid_phase) + db_con.execute(sql_liquid_unit_miles) + + # solid unit-miles for completeness + sql_solid_unit_miles = """ -- total solid unit-miles by mode + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + "{}-miles", + network_source_id, + sum(commodity_flow*miles), + '{}-miles', + "" + from optimal_route_segments + where units = '{}' + group by commodity_name, network_source_id + ;""".format(the_scenario.default_units_solid_phase, the_scenario.default_units_solid_phase, the_scenario.default_units_solid_phase) + db_con.execute(sql_solid_unit_miles) + + logger.debug("start: summarize dollar costs") + # dollar cost and routing cost by mode + # multiply the mileage by the flow + sql_dollar_costs = """ -- total dollar_cost + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + "dollar_cost", + network_source_id, + sum(commodity_flow*link_dollar_cost), + 'USD', + "" + from optimal_route_segments + group by commodity_name, network_source_id + ;""" + db_con.execute(sql_dollar_costs) + + logger.debug("start: summarize routing costs") + sql_routing_costs = """ -- total routing_cost + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + "routing_cost", + network_source_id, + sum(commodity_flow*link_routing_cost), + 'USD', + "" + from optimal_route_segments + group by commodity_name, network_source_id + ;""" + db_con.execute(sql_routing_costs) + + # loads by mode and commodity + # use artificial links =1 and = 2 to calculate loads per loading event + # (e.g. leaving a facility or switching modes at intermodal facility) + + sql_mode_commodity = "select network_source_id, commodity_name from optimal_route_segments group by network_source_id, commodity_name;" + db_cur = db_con.execute(sql_mode_commodity) + mode_and_commodity_list = db_cur.fetchall() + + attributes_dict = ftot_supporting_gis.get_commodity_vehicle_attributes_dict(the_scenario, logger) + + logger.debug("start: summarize vehicle loads") + for row in mode_and_commodity_list: + mode = row[0] + if 'pipeline_crude' in mode: + measure_name = "pipeline_crude_mvts" + if 'pipeline_prod' in mode: + measure_name = "pipeline_prod_mvts" + if "road" in mode: + measure_name = "truck_loads" + if "rail" in mode: + measure_name = "rail_cars" + if "water" in mode: + measure_name = "barge_loads" + commodity_name = row[1] + vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude + sql_vehicle_load = """ -- total loads by mode and commodity + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'vehicles', + network_source_id, + round(sum(commodity_flow/{}/2)), + '{}', -- units + '' + from optimal_route_segments + where (artificial = 1 or artificial = 2) and network_source_id = '{}' and commodity_name = '{}' + group by commodity_name, network_source_id + ;""".format(vehicle_payload, measure_name, mode, commodity_name) + db_con.execute(sql_vehicle_load) + + # VMT + # Do not report for pipeline + + logger.debug("start: summarize VMT") + for row in mode_and_commodity_list: + mode = row[0] + commodity_name = row[1] + + if 'pipeline' in mode: + pass + else: + vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude + # vmt is loads multiplied by miles + # we know the flow on a link we can calculate the loads on that link + # and multiply by the miles to get VMT. + sql_vmt = """ -- VMT by mode and commodity + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'vmt', + '{}', + sum(commodity_flow * miles/{}), + 'VMT', + "" + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' + group by commodity_name, network_source_id + ;""".format(mode, vehicle_payload, mode, commodity_name) + db_con.execute(sql_vmt) + + # CO2-- convert commodity and miles to CO2 + # trucks is a bit different than rail, and water + # fclass = 1 for interstate + # rural_urban_code == 99999 for rural, all else urban + # road CO2 emission factors in g/mi + + logger.debug("start: summarize CO2") + fclass_and_urban_code = {} + fclass_and_urban_code["CO2urbanRestricted"] = {'where': "urban_rural_code < 99999 and fclass = 1"} + fclass_and_urban_code["CO2urbanUnrestricted"] = {'where': "urban_rural_code < 99999 and fclass <> 1"} + fclass_and_urban_code["CO2ruralRestricted"] = {'where': "urban_rural_code = 99999 and fclass = 1"} + fclass_and_urban_code["CO2ruralUnrestricted"] = {'where': "urban_rural_code = 99999 and fclass <> 1"} + + for row in mode_and_commodity_list: + mode = row[0] + commodity_name = row[1] + if 'road' == mode: + vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude + for road_co2_measure_name in fclass_and_urban_code: + co2_val = attributes_dict[commodity_name][mode][road_co2_measure_name].magnitude #g/mi + where_clause = fclass_and_urban_code[road_co2_measure_name]['where'] + sql_road_co2 = """ -- CO2 for road by fclass and urban_rural_code + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + '{}', -- measure_name + '{}', -- mode + sum(commodity_flow * miles * {} /{}), -- value (VMT * co2 scaler) + 'grams', -- units + '' + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' and {} -- additional where_clause + group by commodity_name, network_source_id + ;""".format(road_co2_measure_name, mode, co2_val, vehicle_payload, mode, commodity_name, where_clause) + db_con.execute(sql_road_co2) + + # now total the road co2 values + sql_co2_road_total = """insert into optimal_scenario_results + select + 'commodity_summary', + commodity, + NULL, + 'co2', + mode, + sum(osr.value), + units, + "" + from optimal_scenario_results osr + where osr.measure like 'CO2%ed' + group by commodity + ;""" + db_con.execute(sql_co2_road_total) + + # now delete the intermediate records + sql_co2_delete = """delete + from optimal_scenario_results + where measure like 'CO2%ed' + ;""" + db_con.execute(sql_co2_delete) + + else: + co2_emissions = attributes_dict[commodity_name][mode]['CO2_Emissions'].magnitude #g/ton/mi + sql_co2 = """ -- CO2 by mode and commodity + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'co2', + '{}', + sum(commodity_flow * miles * {}), + 'grams', + "" + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' + group by commodity_name, network_source_id + ;""".format(mode, co2_emissions, mode, commodity_name) + db_con.execute(sql_co2) + + # Fuel burn + # covert VMT to fuel burn + # truck, rail, barge. no pipeline + # Same as VMT but divide result by fuel efficiency + + logger.debug("start: summarize vehicle loads") + for row in mode_and_commodity_list: + mode = row[0] + commodity_name = row[1] + if 'pipeline' in mode: + pass + else: + vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude + fuel_efficiency = attributes_dict[commodity_name][mode]['Fuel_Efficiency'].magnitude #mi/gal + + # vmt is loads multiplied by miles + # we know the flow on a link we can calculate the loads on that link + # and multiply by the miles to get VMT. Then divide by fuel efficiency to get fuel burn + sql_fuel_burn = """ -- Fuel burn by mode and commodity + insert into optimal_scenario_results + select + 'commodity_summary', + commodity_name, + NULL, + 'fuel_burn', + '{}', + sum(commodity_flow * miles/{}/{}), + 'Gallons', + '' + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' + group by commodity_name, network_source_id + ;""".format(mode, vehicle_payload, fuel_efficiency, mode, commodity_name) + db_con.execute(sql_fuel_burn) + + # Destination Deliveries + # look at artificial = 1 and join on facility_name or d_location + # compare optimal flow at the facility (flow on all links) + # to the destination demand in the facility commodities table. + # report % fulfillment by commodity + # Destination report + + logger.debug("start: summarize supply and demand") + sql_destination_demand = """insert into optimal_scenario_results + select + "facility_summary", + c.commodity_name, + f.facility_name, + "destination_demand_total", + "total", + fc.quantity, + fc.units, + '' + from facility_commodities fc + join facilities f on f.facility_id = fc.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where fti.facility_type = 'ultimate_destination' + order by facility_name + ;""" + + # RMP supplier report + sql_rmp_supply = """insert into optimal_scenario_results + select + "facility_summary", + c.commodity_name, + f.facility_name, + "rmp_supply_total", + "total", + fc.quantity, + fc.units, + '' + from facility_commodities fc + join facilities f on f.facility_id = fc.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where fti.facility_type = 'raw_material_producer' + order by facility_name + ;""" + + # initialize SQL queries that vary based on whether routes are used or not + logger.debug("start: summarize optimal supply and demand") + if the_scenario.ndrOn: + sql_destination_demand_optimal = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "destination_demand_optimal", + ne.mode_source, + sum(ov.variable_value), + ov.units, + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'ultimate_destination' and fc.commodity_id = ov.commodity_id + group by ov.d_facility, ov.commodity_name, ne.mode_source + ;""" + + sql_destination_demand_optimal_frac = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "destination_demand_optimal_frac", + ne.mode_source, + (sum(ov.variable_value) / fc.quantity), + "fraction", + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where ov.d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'ultimate_destination' and fc.commodity_id = ov.commodity_id + group by ov.d_facility, ov.commodity_name, ne.mode_source + ;""" + + # RMP supplier report + sql_rmp_supply_optimal = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "rmp_supply_optimal", + ne.mode_source, + sum(ov.variable_value), + ov.units, + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'raw_material_producer' and fc.commodity_id = ov.commodity_id + group by ov.o_facility, ov.commodity_name, ne.mode_source + ;""" + + # RMP supplier report + sql_rmp_supply_optimal_frac = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "rmp_supply_optimal_frac", + ne.mode_source, + --sum(ov.variable_value) as optimal_flow, + --fc.quantity as available_supply, + (sum(ov.variable_value) / fc.quantity), + "fraction", + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'raw_material_producer' and fc.commodity_id = ov.commodity_id + group by ov.o_facility, ov.commodity_name, ne.mode_source + ;""" + # Processor report + sql_processor_output = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "processor_output", + ne.mode_source, + (sum(ov.variable_value)), + ov.units, + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.first_nx_edge_id + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where ov.o_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id + group by ov.o_facility, ov.commodity_name, ne.mode_source + ;""" + + sql_processor_input = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "processor_input", + ne.mode_source, + (sum(ov.variable_value)), + ov.units, + '' + from optimal_variables ov + join edges e on e.edge_id = ov.var_id + join route_reference rr on rr.route_id = e.route_id + join networkx_edges ne on ne.edge_id = rr.last_nx_edge_id + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where ov.d_facility > 0 and ov.edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id + group by ov.d_facility, ov.commodity_name, ne.mode_source + ;""" + else : + sql_destination_demand_optimal = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "destination_demand_optimal", + mode, + sum(ov.variable_value), + ov.units, + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'ultimate_destination' + group by d_facility, mode + ;""" + + sql_destination_demand_optimal_frac = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "destination_demand_optimal_frac", + mode, + (sum(ov.variable_value) / fc.quantity), + "fraction", + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'ultimate_destination' + group by d_facility, mode + ;""" + + # RMP optimal supply + sql_rmp_supply_optimal = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "rmp_supply_optimal", + mode, + sum(ov.variable_value), + ov.units, + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'raw_material_producer' + group by o_facility, mode + ;""" + + # RMP supplier report + sql_rmp_supply_optimal_frac = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "rmp_supply_optimal_frac", + mode, + --sum(ov.variable_value) as optimal_flow, + --fc.quantity as available_supply, + (sum(ov.variable_value) / fc.quantity), + "fraction", + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'raw_material_producer' + group by o_facility, mode + ;""" + + # Processor report + sql_processor_output = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.o_facility, + "processor_output", + mode, + (sum(ov.variable_value)), + ov.units, + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.o_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where o_facility > 0 and edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id + group by o_facility, mode, ov.commodity_name + ;""" + + sql_processor_input = """insert into optimal_scenario_results + select + "facility_summary", + ov.commodity_name, + ov.d_facility, + "processor_input", + mode, + (sum(ov.variable_value)), + ov.units, + '' + from optimal_variables ov + join facilities f on f.facility_name = ov.d_facility + join facility_commodities fc on fc.facility_id = f.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where d_facility > 0 and edge_type = 'transport' and fti.facility_type = 'processor' and fc.commodity_id = ov.commodity_id + group by d_facility, mode, ov.commodity_name + ;""" + + db_con.execute(sql_destination_demand) + db_con.execute(sql_destination_demand_optimal) + db_con.execute(sql_destination_demand_optimal_frac) + db_con.execute(sql_rmp_supply) + db_con.execute(sql_rmp_supply_optimal) + db_con.execute(sql_rmp_supply_optimal_frac) + db_con.execute(sql_processor_output) + db_con.execute(sql_processor_input) + + # measure totals + sql_total = """insert into optimal_scenario_results + select table_name, commodity, facility_name, measure, "_total" as mode, sum(value), units, notes + from optimal_scenario_results + group by table_name, commodity, facility_name, measure + ;""" + db_con.execute(sql_total) + + logger.debug("finish: make_optimal_scenario_results_db()") + + +# =================================================================================================== + + +def generate_scenario_summary(the_scenario, logger): + logger.info("starting generate_scenario_summary") + + # query the optimal scenario results table and report out the results + with sqlite3.connect(the_scenario.main_db) as db_con: + + sql = "select * from optimal_scenario_results order by table_name, commodity, measure, mode;" + db_cur = db_con.execute(sql) + data = db_cur.fetchall() + + for row in data: + if row[2] == None: # these are the commodity summaries with no facility name + logger.result('{}_{}_{}_{}: \t {:,.2f} : \t {}'.format(row[0].upper(), row[3].upper(), row[1].upper(), row[4].upper(), row[5], row[6])) + else: # these are the commodity summaries with no facility name + logger.result('{}_{}_{}_{}_{}: \t {:,.2f} : \t {}'.format(row[0].upper(), row[2].upper(), row[3].upper(), row[1].upper(), row[4].upper(), row[5], row[6])) + + logger.debug("finish: generate_scenario_summary()") + + +# =================================================================================================== + +def detailed_emissions_setup(the_scenario, logger): + + logger.info("START: detailed_emissions_setup") + + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_con.executescript(""" + drop table if exists detailed_emissions; + create table detailed_emissions( + commodity text, + mode text, + measure text, + value text, + units text, + constraint unique_elements unique(commodity, mode, measure)) + ;""") + + attributes_dict = ftot_supporting_gis.get_commodity_vehicle_attributes_dict(the_scenario, logger, EmissionsWarning=True) + + sql_mode_commodity = "select network_source_id, commodity_name from optimal_route_segments group by network_source_id, commodity_name;" + db_cur = db_con.execute(sql_mode_commodity) + mode_and_commodity_list = db_cur.fetchall() + + + # Convert commodity and miles to pollutant totals + # ROAD: + # fclass = 1 for interstate + # rural_urban_code == 99999 for rural, all else urban + # road CO2 emission factors in g/mi + + fclass_and_urban_code = {} + fclass_and_urban_code["Urban_Restricted"] = {'where': "urban_rural_code < 99999 and fclass = 1"} + fclass_and_urban_code["Urban_Unrestricted"] = {'where': "urban_rural_code < 99999 and fclass <> 1"} + fclass_and_urban_code["Rural_Restricted"] = {'where': "urban_rural_code = 99999 and fclass = 1"} + fclass_and_urban_code["Rural_Unrestricted"] = {'where': "urban_rural_code = 99999 and fclass <> 1"} + + for row in mode_and_commodity_list: + mode = row[0] + commodity_name = row[1] + for attr in attributes_dict[commodity_name][mode]: + if attr not in ['co','co2e','ch4','n2o','nox','pm10','pm2.5','voc']: + continue + elif mode == 'road': + vehicle_payload = attributes_dict[commodity_name][mode]['Load'].magnitude + for road_measure_name in fclass_and_urban_code: + ef = attributes_dict[commodity_name][mode][attr][road_measure_name].magnitude #g/mi + where_clause = fclass_and_urban_code[road_measure_name]['where'] + sql_road = """ -- {} for road by fclass and urban_rural_code + insert into detailed_emissions + select + commodity_name, + '{}', + '{}', + sum(commodity_flow * miles * {} /{}), + 'grams' + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' and {} + group by commodity_name, network_source_id + ;""".format(attr, mode, road_measure_name, ef, vehicle_payload, mode, commodity_name, where_clause) + db_con.execute(sql_road) + + # now total the road values + sql_road_total = """insert into detailed_emissions + select + commodity, + mode, + '{}', + sum(em.value), + units + from detailed_emissions em + where em.measure like '%stricted' + group by commodity + ;""".format(attr) + db_con.execute(sql_road_total) + + # now delete the intermediate records + sql_road_delete = """delete + from detailed_emissions + where measure like '%stricted' + ;""" + db_con.execute(sql_road_delete) + + else: + ef = attributes_dict[commodity_name][mode][attr].magnitude #g/ton/mi + sql_em = """ -- {} by mode and commodity + insert into detailed_emissions + select + commodity_name, + '{}', + '{}', + sum(commodity_flow * miles * {}), + 'grams' + from optimal_route_segments + where network_source_id = '{}' and commodity_name = '{}' + group by commodity_name, network_source_id + ;""".format(attr, mode, attr, ef, mode, commodity_name) + db_con.execute(sql_em) + + +# =================================================================================================== + + +def generate_artificial_link_summary(the_scenario, logger): + logger.info("starting generate_artificial_link_summary") + + # query the facility and network tables for artificial links and report out the results + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select fac.facility_name, fti.facility_type, ne.mode_source, round(ne.miles, 3) as miles + from facilities fac + left join facility_type_id fti on fac.facility_type_id = fti.facility_type_id + left join networkx_nodes nn on fac.location_id = nn.location_id + left join networkx_edges ne on nn.node_id = ne.from_node_id + where nn.location_1 like '%OUT%' + and ne.artificial = 1 + ;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + + artificial_links = {} + for row in db_data: + facility_name = row[0] + facility_type = row[1] + mode_source = row[2] + link_length = row[3] + + if facility_name not in artificial_links: + # add new facility to dictionary and start list of artificial links by mode + artificial_links[facility_name] = {'fac_type': facility_type, 'link_lengths': {}} + + if mode_source not in artificial_links[facility_name]['link_lengths']: + artificial_links[facility_name]['link_lengths'][mode_source] = link_length + else: + # there should only be one artificial link for a facility for each mode + error = "Multiple artificial links should not be found for a single facility for a particular mode." + logger.error(error) + raise Exception(error) + + # create structure for artificial link csv table + output_table = {'facility_name': [], 'facility_type': []} + for permitted_mode in the_scenario.permittedModes: + output_table[permitted_mode] = [] + + # iterate through every facility, add a row to csv table + for k in artificial_links: + output_table['facility_name'].append(k) + output_table['facility_type'].append(artificial_links[k]['fac_type']) + for permitted_mode in the_scenario.permittedModes: + if permitted_mode in artificial_links[k]['link_lengths']: + output_table[permitted_mode].append(artificial_links[k]['link_lengths'][permitted_mode]) + else: + output_table[permitted_mode].append('NA') + + # print artificial link data for each facility to file in debug folder + import csv + with open(os.path.join(the_scenario.scenario_run_directory, "debug", 'artificial_links.csv'), 'w', newline='') as f: + writer = csv.writer(f) + output_fields = ['facility_name', 'facility_type'] + the_scenario.permittedModes + writer.writerow(output_fields) + writer.writerows(zip(*[output_table[key] for key in output_fields])) + + logger.debug("finish: generate_artificial_link_summary()") + + +# ============================================================================================== + + +def db_report_commodity_utilization(the_scenario, logger): + logger.info("start: db_report_commodity_utilization") + + # This query pulls the total quantity flowed of each commodity from the optimal scenario results (osr) table. + # It groups by commodity_name, facility type, and units. The io field is included to help the user + # determine the potential supply, demand, and processing utilization in the scenario. + # ----------------------------------- + + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select c.commodity_name, + fti.facility_type, + io, + IFNULL((select osr.value + from optimal_scenario_results osr + where osr.commodity = c.commodity_name + and osr.measure ='total_flow' + and osr.mode = '_total'),-9999) optimal_flow, + c.units + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility == 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc + ;""" + db_cur = db_con.execute(sql) + + db_data = db_cur.fetchall() + logger.result("-------------------------------------------------------------------") + logger.result("Scenario Total Flow of Supply and Demand") + logger.result("-------------------------------------------------------------------") + logger.result("total utilization is defined as (total flow / net available)") + logger.result("commodity_name | facility_type | io | optimal_flow | units ") + logger.result("---------------|---------------|----|---------------|----------") + for row in db_data: + logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], + row[4])) + logger.result("-------------------------------------------------------------------") + + # This query compares the optimal flow to the net available quantity of material in the scenario. + # It groups by commodity_name, facility type, and units. The io field is included to help the user + # determine the potential supply, demand, and processing utilization in the scenario. + # ----------------------------------- + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select c.commodity_name, + fti.facility_type, + io, + IFNULL(round((select osr.value + from optimal_scenario_results osr + where osr.commodity = c.commodity_name + and osr.measure ='total_flow' + and osr.mode = '_total') / sum(fc.quantity),2),-9999) Utilization, + 'fraction' + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility == 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc + ;""" + db_cur = db_con.execute(sql) + + db_data = db_cur.fetchall() + logger.result("-------------------------------------------------------------------") + logger.result("Scenario Total Utilization of Supply and Demand") + logger.result("-------------------------------------------------------------------") + logger.result("total utilization is defined as (total flow / net available)") + logger.result("commodity_name | facility_type | io | utilization | units ") + logger.result("---------------|---------------|----|---------------|----------") + for row in db_data: + logger.result("{:15.15} {:15.15} {:4.1} {:15,.1f} {:15.10}".format(row[0], row[1], row[2], row[3], + row[4])) + logger.result("-------------------------------------------------------------------") + + +# =================================================================================================== + + +def make_optimal_route_segments_featureclass_from_db(the_scenario, + logger): + + logger.info("starting make_optimal_route_segments_featureclass_from_db") + + # create the segments layer + # ------------------------- + optimized_route_segments_fc = os.path.join(the_scenario.main_gdb, "optimized_route_segments") + + if arcpy.Exists(optimized_route_segments_fc): + arcpy.Delete_management(optimized_route_segments_fc) + logger.debug("deleted existing {} layer".format(optimized_route_segments_fc)) + + arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments", \ + "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ, "#", "0", "0", "0") + + arcpy.AddField_management(optimized_route_segments_fc, "ROUTE_TYPE", "TEXT", "#", "#", "25", "#", "NULLABLE", "NON_REQUIRED", "#") + arcpy.AddField_management(optimized_route_segments_fc, "FTOT_RT_ID", "LONG") + arcpy.AddField_management(optimized_route_segments_fc, "FTOT_RT_ID_VARIANT", "LONG") + arcpy.AddField_management(optimized_route_segments_fc, "NET_SOURCE_NAME", "TEXT") + arcpy.AddField_management(optimized_route_segments_fc, "NET_SOURCE_OID", "LONG") + arcpy.AddField_management(optimized_route_segments_fc, "ARTIFICIAL", "SHORT") + + # if from pos = 0 then traveresed in direction of underlying arc, otherwise traversed against the flow of the arc. + # used to determine if from_to_dollar cost should be used or to_from_dollar cost should be used. + arcpy.AddField_management(optimized_route_segments_fc, "FromPosition", "DOUBLE") # mnp 8/30/18 -- deprecated + arcpy.AddField_management(optimized_route_segments_fc, "FromJunctionID", "DOUBLE") # mnp - 060116 - added this for processor siting step + arcpy.AddField_management(optimized_route_segments_fc, "TIME_PERIOD", "TEXT") + arcpy.AddField_management(optimized_route_segments_fc, "COMMODITY", "TEXT") + arcpy.AddField_management(optimized_route_segments_fc, "COMMODITY_FLOW", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "VOLUME", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "CAPACITY", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "CAPACITY_MINUS_VOLUME", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "UNITS", "TEXT") + arcpy.AddField_management(optimized_route_segments_fc, "MILES", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "PHASE_OF_MATTER", "TEXT") + arcpy.AddField_management(optimized_route_segments_fc, "LINK_ROUTING_COST", "FLOAT") + arcpy.AddField_management(optimized_route_segments_fc, "LINK_DOLLAR_COST", "FLOAT") + + # get a list of the modes used in the optimal route_segments stored in the db. + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_cur = db_con.cursor() + + # get a list of the source_ids and convert to modes: + sql = "select distinct network_source_id from optimal_route_segments;" + db_cur = db_con.execute(sql) + rows = db_cur.fetchall() + + mode_source_list = [] + for row in rows: + mode_source_list.append(row[0]) + + logger.debug("List of modes used in the optimal solution: {}".format(mode_source_list)) + + # iterate through the segment_dict_by_source_name + # and get the segments to insert into the optimal segments layer + # --------------------------------------------------------------------- + + optimized_route_seg_flds = ("SHAPE@", "FTOT_RT_ID", "FTOT_RT_ID_VARIANT", "ROUTE_TYPE", "NET_SOURCE_NAME", + "NET_SOURCE_OID", "FromPosition", "FromJunctionID", "MILES", "TIME_PERIOD", "COMMODITY", "COMMODITY_FLOW", + "VOLUME", "CAPACITY", "CAPACITY_MINUS_VOLUME", "UNITS", "PHASE_OF_MATTER", "ARTIFICIAL", "LINK_ROUTING_COST", "LINK_DOLLAR_COST") + + for network_source in mode_source_list: + if network_source == 'intermodal': + logger.debug("network_source is: {}. can't flatten this, so skipping.".format(network_source)) + continue + + network_link_counter = 0 + network_link_coverage_counter = 0 + network_source_fc = os.path.join(the_scenario.main_gdb, "network", network_source) + + sql = """select DISTINCT + scenario_rt_id, + NULL, + network_source_id, + network_source_oid, + NULL, + NULL, + time_period, + commodity_name, + commodity_flow, + volume, + capacity, + capacity_minus_volume, + units, + phase_of_matter, + miles, + route_type, + artificial, + link_dollar_cost, + link_routing_cost + from optimal_route_segments + where network_source_id = '{}' + ;""".format(network_source) + + logger.debug("starting the execute for the {} mode".format(network_source)) + db_cur = db_con.execute(sql) + logger.debug("done with the execute") + logger.debug("starting fetch all for the {} mode".format(network_source)) + rows = db_cur.fetchall() + logger.debug("done with the fetchall") + + optimal_route_segments_dict = {} + + logger.debug("starting to build the dict for {}".format(network_source)) + for row in rows: + if row[3] not in optimal_route_segments_dict: + optimal_route_segments_dict[row[3]] = [] + network_link_coverage_counter += 1 + + route_id = row[0] + route_id_variant = row[1] + network_source_id = row[2] + network_object_id = row[3] + from_position = row[4] + from_junction_id = row[5] + time_period = row[6] + commodity = row[7] + commodity_flow = row[8] + volume = row[9] + capacity = row[10] + capacity_minus_volume= row[11] + units = row[12] + phase_of_matter = row[13] + miles = row[14] + route_type = row[15] + artificial = row[16] + link_dollar_cost = row[17] + link_routing_cost = row[18] + + optimal_route_segments_dict[row[3]].append([route_id, route_id_variant, network_source_id, + network_object_id, from_position, from_junction_id, time_period, + commodity, commodity_flow, volume, capacity, + capacity_minus_volume, units, phase_of_matter, miles, + route_type, artificial, link_dollar_cost, link_routing_cost]) + + logger.debug("done building the dict for {}".format(network_source)) + + logger.debug("starting the search cursor") + with arcpy.da.SearchCursor(network_source_fc, ["OBJECTID", "SHAPE@"]) as search_cursor: + logger.info("start: looping through the {} mode".format(network_source)) + for row in search_cursor: + network_link_counter += 1 + object_id = row[0] + geom = row[1] + + if object_id not in optimal_route_segments_dict: + continue + else: + for segment_info in optimal_route_segments_dict[object_id]: + (route_id, route_id_variant, network_source_id, network_object_id, from_position, + from_junction_id, time_period, commodity, commodity_flow, volume, capacity, capacity_minus_volume, + units, phase_of_matter, miles, route_type, artificial, link_dollar_cost, link_routing_cost) = segment_info + with arcpy.da.InsertCursor(optimized_route_segments_fc, optimized_route_seg_flds) as insert_cursor: + insert_cursor.insertRow([geom, route_id, route_id_variant, route_type, network_source_id, + network_object_id, from_position, from_junction_id, + miles, time_period, commodity, commodity_flow, volume, capacity, + capacity_minus_volume, units, phase_of_matter, artificial, + link_routing_cost, link_dollar_cost]) + + logger.debug("finish: looping through the {} mode".format(network_source)) + logger.info("mode: {} coverage: {:,.1f} - total links: {} , total links used: {}".format(network_source, + 100.0*(int(network_link_coverage_counter)/float(network_link_counter)), network_link_counter, network_link_coverage_counter )) + + +# ====================================================================================================================== + + +def add_fclass_and_urban_code(the_scenario, logger): + + logger.info("starting add_fclass_and_urban_code") + scenario_gdb = the_scenario.main_gdb + optimized_route_segments_fc = os.path.join(scenario_gdb, "optimized_route_segments") + + # build up dictionary network links we are interested in + # ---------------------------------------------------- + + network_source_oid_dict = {} + network_source_oid_dict['road'] = {} + + with arcpy.da.SearchCursor(optimized_route_segments_fc, ("NET_SOURCE_NAME", "NET_SOURCE_OID")) as search_cursor: + for row in search_cursor: + if row[0] == 'road': + network_source_oid_dict['road'][row[1]] = True + + # iterate different network layers (i.e. rail, road, water, pipeline) + # and get needed info into the network_source_oid_dict dict + # ----------------------------------------------------------- + + road_fc = os.path.join(scenario_gdb, "network", "road") + flds = ("OID@", "FCLASS", "URBAN_CODE") + with arcpy.da.SearchCursor(road_fc, flds) as search_cursor: + for row in search_cursor: + if row[0] in network_source_oid_dict['road']: + network_source_oid_dict['road'][row[0]] = [row[1], row[2]] + + # add fields to hold data from network links + # ------------------------------------------ + + # FROM ROAD NET ONLY + arcpy.AddField_management(optimized_route_segments_fc, "ROAD_FCLASS", "SHORT") + arcpy.AddField_management(optimized_route_segments_fc, "URBAN_CODE", "LONG") + + # set network link related values in optimized_route_segments layer + # ----------------------------------------------------------------- + + flds = ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ROAD_FCLASS", "URBAN_CODE"] + + # initialize a list for the update + + update_list = [] + net_oid_seen = {} + + with arcpy.da.UpdateCursor(optimized_route_segments_fc, flds) as update_cursor: + + for row in update_cursor: + + net_source = row[0] + net_oid = row[1] + + if net_source == 'road': + data = network_source_oid_dict[net_source][net_oid] + row[2] = data[0] # fclass + row[3] = data[1] # urban_code + + update_cursor.updateRow(row) + + # Just adding the fclass and urban_rural_code to the list once per net_source_oid + if net_oid not in net_oid_seen: + net_oid_seen[net_oid] = 0 + update_list.append([row[2], row[3], net_source, net_oid]) + net_oid_seen[net_oid] += 1 + + logger.info("starting the execute many to update the list of optimal_route_segments") + with sqlite3.connect(the_scenario.main_db) as db_con: + db_con.execute("CREATE INDEX if not exists network_index ON optimal_route_segments(network_source_id, network_source_oid, commodity_name, artificial)") + update_sql = """ + UPDATE optimal_route_segments + set fclass = ?, urban_rural_code = ? + where network_source_id = ? and network_source_oid = ? + ;""" + + db_con.executemany(update_sql, update_list) + db_con.commit() + + +# ====================================================================================================================== + + +def dissolve_optimal_route_segments_feature_class_for_mapping(the_scenario, logger): + + # Make a dissolved version of fc for mapping aggregate flows + logger.info("starting dissolve_optimal_route_segments_feature_class_for_mapping") + + scenario_gdb = the_scenario.main_gdb + optimized_route_segments_fc = os.path.join(scenario_gdb, "optimized_route_segments") + + arcpy.env.workspace = scenario_gdb + + if arcpy.Exists("optimized_route_segments_dissolved"): + arcpy.Delete_management("optimized_route_segments_dissolved") + + arcpy.MakeFeatureLayer_management(optimized_route_segments_fc, "segments_lyr") + result = arcpy.GetCount_management("segments_lyr") + count = str(result.getOutput(0)) + + if arcpy.Exists("optimized_route_segments_dissolved_tmp"): + arcpy.Delete_management("optimized_route_segments_dissolved_tmp") + + if arcpy.Exists("optimized_route_segments_split_tmp"): + arcpy.Delete_management("optimized_route_segments_split_tmp") + + if arcpy.Exists("optimized_route_segments_dissolved_tmp2"): + arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") + + if int(count) > 0: + + # Dissolve + arcpy.Dissolve_management(optimized_route_segments_fc, "optimized_route_segments_dissolved_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", "PHASE_OF_MATTER", "UNITS"], + [['COMMODITY_FLOW', 'SUM']], "SINGLE_PART", "DISSOLVE_LINES") + + if arcpy.CheckProduct("ArcInfo") == "Available": + # Second dissolve needed to accurately show aggregate pipeline flows + arcpy.FeatureToLine_management("optimized_route_segments_dissolved_tmp", + "optimized_route_segments_split_tmp") + + arcpy.AddGeometryAttributes_management("optimized_route_segments_split_tmp", "LINE_START_MID_END") + + arcpy.Dissolve_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2", + ["NET_SOURCE_NAME", "Shape_Length", "MID_X", "MID_Y", "ARTIFICIAL", + "PHASE_OF_MATTER", "UNITS"], + [["SUM_COMMODITY_FLOW", "SUM"]], "SINGLE_PART", "DISSOLVE_LINES") + + arcpy.AddField_management("optimized_route_segments_dissolved_tmp2", "SUM_COMMODITY_FLOW", "DOUBLE") + arcpy.CalculateField_management("optimized_route_segments_dissolved_tmp2", "SUM_COMMODITY_FLOW", + "!SUM_SUM_COMMODITY_FLOW!", "PYTHON_9.3") + arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "SUM_SUM_COMMODITY_FLOW") + arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "MID_X") + arcpy.DeleteField_management("optimized_route_segments_dissolved_tmp2", "MID_Y") + + else: + # Doing it differently because feature to line isn't available without an advanced license + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. A modification to the " + "dissolve_optimal_route_segments_feature_class_for_mapping method is necessary") + + # Create the fc + arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments_split_tmp", + "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ) + + arcpy.AddField_management("optimized_route_segments_split_tmp", "NET_SOURCE_NAME", "TEXT") + arcpy.AddField_management("optimized_route_segments_split_tmp", "NET_SOURCE_OID", "LONG") + arcpy.AddField_management("optimized_route_segments_split_tmp", "ARTIFICIAL", "SHORT") + arcpy.AddField_management("optimized_route_segments_split_tmp", "UNITS", "TEXT") + arcpy.AddField_management("optimized_route_segments_split_tmp", "PHASE_OF_MATTER", "TEXT") + arcpy.AddField_management("optimized_route_segments_split_tmp", "SUM_COMMODITY_FLOW", "DOUBLE") + + # Go through the pipeline segments separately + tariff_segment_dict = defaultdict(float) + with arcpy.da.SearchCursor("optimized_route_segments_dissolved_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", "PHASE_OF_MATTER", "UNITS", + "SUM_COMMODITY_FLOW", "SHAPE@"]) as search_cursor: + for row1 in search_cursor: + if 'pipeline' in row1[0]: + # Must not be artificial, otherwise pass the link through + if row1[2] == 0: + # Capture the tariff ID so that we can link to the segments + mode = row1[0] + with arcpy.da.SearchCursor(mode, ["OBJECTID", "Tariff_ID", "SHAPE@"]) \ + as search_cursor_2: + for row2 in search_cursor_2: + if row1[1] == row2[0]: + tariff_id = row2[1] + mode = row1[0].strip("rts") + with arcpy.da.SearchCursor(mode + "sgmts", ["MASTER_OID", "Tariff_ID", "SHAPE@"]) \ + as search_cursor_3: + for row3 in search_cursor_3: + if tariff_id == row3[1]: + # keying off master_oid, net_source_name, phase of matter, units + shape + tariff_segment_dict[(row3[0], row1[0], row1[3], row1[4], row3[2])] += row1[5] + else: + with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", + "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"]) \ + as insert_cursor: + insert_cursor.insertRow([row1[0], row1[1], row1[2], row1[3], row1[4], row1[5], row1[6]]) + # If it isn't pipeline just pass the data through. + else: + with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", + "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"])\ + as insert_cursor: + insert_cursor.insertRow([row1[0], row1[1], row1[2], row1[3], row1[4], row1[5], row1[6]]) + + # Now that pipeline segment dictionary is built, get the pipeline segments in there as well + for master_oid, net_source_name, phase_of_matter, units, shape in tariff_segment_dict: + commodity_flow = tariff_segment_dict[master_oid, net_source_name, phase_of_matter, units, shape] + with arcpy.da.InsertCursor("optimized_route_segments_split_tmp", + ["NET_SOURCE_NAME", "NET_SOURCE_OID", "ARTIFICIAL", + "PHASE_OF_MATTER", "UNITS", "SUM_COMMODITY_FLOW", "SHAPE@"]) \ + as insert_cursor: + insert_cursor.insertRow([net_source_name, master_oid, 0, phase_of_matter, units, commodity_flow, + shape]) + # No need for dissolve because dictionaries have already summed flows + arcpy.Copy_management("optimized_route_segments_split_tmp", "optimized_route_segments_dissolved_tmp2") + + # Sort for mapping order + arcpy.AddField_management("optimized_route_segments_dissolved_tmp2", "SORT_FIELD", "SHORT") + arcpy.MakeFeatureLayer_management("optimized_route_segments_dissolved_tmp2", "dissolved_segments_lyr") + arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'road'") + arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 1, "PYTHON_9.3") + arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'rail'") + arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 2, "PYTHON_9.3") + arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", "NET_SOURCE_NAME = 'water'") + arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 3, "PYTHON_9.3") + arcpy.SelectLayerByAttribute_management("dissolved_segments_lyr", "NEW_SELECTION", + "NET_SOURCE_NAME LIKE 'pipeline%'") + arcpy.CalculateField_management("dissolved_segments_lyr", "SORT_FIELD", 4, "PYTHON_9.3") + + arcpy.Sort_management("optimized_route_segments_dissolved_tmp2", "optimized_route_segments_dissolved", + [["SORT_FIELD", "ASCENDING"]]) + + # Delete temp fc's + arcpy.Delete_management("optimized_route_segments_dissolved_tmp") + arcpy.Delete_management("optimized_route_segments_split_tmp") + arcpy.Delete_management("optimized_route_segments_dissolved_tmp2") + + else: + arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "optimized_route_segments_dissolved", + "POLYLINE", "#", "DISABLED", "DISABLED", ftot_supporting_gis.LCC_PROJ, "#", + "0", "0", "0") + + arcpy.AddField_management("optimized_route_segments_dissolved", "NET_SOURCE_NAME", "TEXT") + arcpy.AddField_management("optimized_route_segments_dissolved", "ARTIFICIAL", "SHORT") + arcpy.AddField_management("optimized_route_segments_dissolved", "UNITS", "TEXT") + arcpy.AddField_management("optimized_route_segments_dissolved", "PHASE_OF_MATTER", "TEXT") + arcpy.AddField_management("optimized_route_segments_dissolved", "SUM_COMMODITY_FLOW", "DOUBLE") + + arcpy.Delete_management("segments_lyr") + diff --git a/program/ftot_processor.py b/program/ftot_processor.py index 0c86b55..1620bd4 100644 --- a/program/ftot_processor.py +++ b/program/ftot_processor.py @@ -1,1190 +1,1190 @@ -# ------------------------------------------------------------------------------ -# Name: ftot_processor.py -# -# Purpose: generates candidates from optimized flow of feedstock to the -# max_raw_material_transport distance -# -# ------------------------------------------------------------------------------ - -import os -import random -import sqlite3 -import datetime -import arcpy -import ftot_supporting -import ftot_supporting_gis -import pdb -from ftot_facilities import get_commodity_id -from ftot_facilities import get_schedule_id -from ftot_pulp import parse_optimal_solution_db -from ftot import Q_ -from six import iteritems - - -# ============================================================================== - - -def clean_candidate_processor_tables(the_scenario, logger): - logger.debug("start: clean_candidate_processor_tables") - - # clean up the tables, then create them. - # ---------------------------------------- - with sqlite3.connect(the_scenario.main_db) as db_con: - db_con.executescript(""" - drop table if exists candidate_process_list; - drop table if exists candidate_process_commodities; - - create table candidate_process_list( - process_id INTEGER PRIMARY KEY, - process_name text, - minsize numeric, - maxsize numeric, - min_max_size_units text, - cost_formula numeric, - cost_formula_units text, - min_aggregation numeric, - schedule_id integer - ); - - - create table candidate_process_commodities( - process_id integer, - io text, - commodity_name text, - commodity_id integer, - quantity numeric, - units text, - phase_of_matter text - ); - """) - - return - - -# ------------------------------------------------------------------------------ - - -def generate_candidate_processor_tables(the_scenario, logger): - logger.info("start: generate_candidate_processor_tables") - # drops, and then creates 2 tables related to candidates: - # (1) candidate_process_list, which creates a unique ID for each process name, - # and records the respective min and max size, units, and cost formula and units. - - # clean up the tables, then create them. - # ---------------------------------------- - clean_candidate_processor_tables(the_scenario, logger) - - # if the user specifies a processor_candidate_slate_data file - # use that to populate the candidate_processor_list and - # candidate_processor_commodities tables. - # - # note: the processors_candidate_slate_data attribute is user specified slates for generic conversion. - # whereas the processors_commodity_data is FTOT generated for specific candidates - if the_scenario.processors_candidate_slate_data != 'None': - - # get the slate from the facility_commodities table - # for all facilities named "candidate%" - # (note the wild card, '%' in the search). - candidate_process_list, candidate_process_commodities = get_candidate_process_data(the_scenario, logger) - - # process the candidate_process_list first so we can assign a process_id - # the method returns a dictionary keyed off the process_name and has process_id as the value. - # ----------------------------------------------------------------------- - process_id_name_dict = populate_candidate_process_list_table(the_scenario, candidate_process_list, logger) - - # process the candidate_process_commodities list. - # ------------------------------------------------ - populate_candidate_process_commodities(the_scenario, candidate_process_commodities, process_id_name_dict, - logger) - - -# ------------------------------------------------------------------------------ - - -def populate_candidate_process_commodities(the_scenario, candidate_process_commodities, process_id_name_dict, logger): - logger.info("start: populate_candidate_process_commodities") - - logger.debug("number of process IDs: {} ".format(len(process_id_name_dict.keys()))) - logger.debug("number of commodity records: {}".format(len(candidate_process_commodities))) - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # iterate through the list and get the process_id and commodity_id - for commodity in candidate_process_commodities: - logger.debug("start: {} ".format(commodity[0])) - - # replace the process_name with process_id - process_name = commodity[0] - process_id = process_id_name_dict[process_name] - commodity[0] = process_id - - # reusing the ftot_facilities.get_commodity_id module. - # so we need to get the dataset into the proper format it is expecting - # to add the commodity if it doesn't exist in the db yet. - commodity_name = commodity[1] - commodity_quantity = commodity[3] - commodity_unit = commodity[4] - commodity_phase = commodity[5] - commodity_max_transport_dist = 'Null' - io = commodity[6] - shared_max_transport_distance = 'N' - processor_max_input = commodity[3] - # empty string for facility type and schedule id because fields are not used - commodity_data = ['', commodity_name, commodity_quantity, commodity_unit, commodity_phase, commodity_max_transport_dist, io, shared_max_transport_distance, processor_max_input, ''] - - # get commodity_id. (adds commodity if it doesn't exist) - commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger) - commodity[2] = commodity_id - - # now do an execute many on the lists for the segments and route_segments table - sql = "insert into candidate_process_commodities (" \ - "process_id, commodity_name, commodity_id, quantity, units, phase_of_matter, io) " \ - "values (?, ?, ?, ?, ?, ?, ?);" - db_con.executemany(sql, candidate_process_commodities) - db_con.commit() - - -# ------------------------------------------------------------------------------ - - -def create_commodity_id_name_dict(the_scenario, logger): - logger.debug("start: create_commodity_id_name_dict") - commodity_id_name_dict = {} - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = "select commodity_name, commodity_id from commodities;" - db_cur = db_con.execute(sql) - data = db_cur.fetchall() - for row in data: - commodity_id_name_dict[row[0]] = row[1] - logger.debug("returning {} commodity id/name records".format(len(commodity_id_name_dict))) - return commodity_id_name_dict - - -# ------------------------------------------------------------------------------ - - -def populate_candidate_process_list_table(the_scenario, candidate_process_list, logger): - logger.debug("start: populate_candidate_process_list_table") - - candidate_process_list_data = [] - - with sqlite3.connect(the_scenario.main_db) as db_con: - for process in candidate_process_list: - min_size = '' - min_size_units = '' - max_size = '' - max_size_units = '' - cost_formula = '' - cost_formula_units = '' - min_aggregation = '' - min_aggregation_units = '' - schedule_name = '' - process_name = process - - for process_property in candidate_process_list[process]: - if 'schedule_name' == process_property[0]: - schedule_name = process_property[1] - schedule_id = get_schedule_id(the_scenario, db_con, schedule_name, logger) - if 'minsize' == process_property[0]: - min_size = process_property[1] - min_size_units = str(process_property[2]) - if 'maxsize' == process_property[0]: - max_size = process_property[1] - max_size_units = str(process_property[2]) - if 'cost_formula' == process_property[0]: - cost_formula = process_property[1] - cost_formula_units = str(process_property[2]) - if 'min_aggregation' == process_property[0]: - min_aggregation = process_property[1] - min_aggregation_units = str(process_property[2]) - - # do some checks to make sure things aren't weird. - # ------------------------------------------------- - if max_size_units != min_size_units: - logger.warning("the units for the max_size and min_size candidate process do not match!") - if max_size_units != min_aggregation_units: - logger.warning("the units for the max_size and min_aggregation candidate process do not match!") - if min_size == '': - logger.warning("the min_size is set to Null") - if max_size == '': - logger.warning("the max_size is set to Null") - if min_aggregation == '': - logger.warning("the min_aggregation was not specified by csv and has been set to 1/4 min_size") - min_aggregation = float(min_size)/4 - if cost_formula == '': - logger.warning("the cost_formula is set to Null") - if cost_formula_units == '': - logger.warning("the cost_formula_units is set to Null") - - # otherwise, build up a list to add to the sql database. - candidate_process_list_data.append( - [process_name, min_size, max_size, max_size_units, cost_formula, cost_formula_units, min_aggregation, schedule_id]) - - # now do an execute many on the lists for the segments and route_segments table - - sql = "insert into candidate_process_list " \ - "(process_name, minsize, maxsize, min_max_size_units, cost_formula, cost_formula_units, " \ - "min_aggregation, schedule_id) values (?, ?, ?, ?, ?, ?, ?, ?);" - db_con.executemany(sql, candidate_process_list_data) - db_con.commit() - - # now return a dictionary of process_id and process_names - process_id_name_dict = {} - sql = "select process_name, process_id from candidate_process_list;" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - for row in db_data: - process_id_name_dict[row[0]] = row[1] - logger.info("note: added {} candidate processes to the candidate_process_list table".format( - len(list(process_id_name_dict.keys())))) - logger.debug("ID || Process Name:") - logger.debug("-----------------------") - for process_name in process_id_name_dict: - logger.debug("{} || {}".format(process_id_name_dict[process_name], process_name)) - return process_id_name_dict - - -# ===================================================================================================================== - - -def get_candidate_process_data(the_scenario, logger): - logger.info("start: get_candidate_process_data") - - from ftot_facilities import load_facility_commodities_input_data - - candidate_process_data = load_facility_commodities_input_data(the_scenario, - the_scenario.processors_candidate_slate_data, logger) - candidate_process_commodities = [] - candidate_process_list = {} - - for facility_name, facility_commodity_list in iteritems(candidate_process_data): - for row in facility_commodity_list: - commodity_name = row[1] - commodity_id = None - quantity = row[2] - units = str(row[3]) - phase_of_matter = row[4] - io = row[6] - schedule_name = row[-1] - - # store the input and output commodities - # in the candidate_process_commodities (CPC) list . - if io == 'i' or io == 'o': - candidate_process_commodities.append( - [facility_name, commodity_name, commodity_id, quantity, units, phase_of_matter, io]) - - # store the facility size and cost information in the - # candidate_process_list (CPL). - else: - if facility_name not in list(candidate_process_list.keys()): - candidate_process_list[facility_name] = [] - # add schedule name to the candidate_process_list array for the facility - candidate_process_list[facility_name].append(["schedule_name", schedule_name]) - - if commodity_name == 'maxsize': - candidate_process_list[facility_name].append(["maxsize", quantity, units]) - elif commodity_name == 'minsize': - candidate_process_list[facility_name].append(["minsize", quantity, units]) - elif commodity_name == 'cost_formula': - candidate_process_list[facility_name].append(["cost_formula", quantity, units]) - elif commodity_name == 'min_aggregation': - candidate_process_list[facility_name].append(["min_aggregation", quantity, units]) - - # log a warning if nothing came back from the query - if len(candidate_process_list) == 0: - logger.warning("the len(candidate_process_list) == 0.") - logger.info("TIP: Make sure the process names in the Processors_Candidate_Commodity_Data " - "are prefixed by the word 'candidate_'. e.g. 'candidate_HEFA'") - - # log a warning if nothing came back from the query - if len(candidate_process_commodities) == 0: - logger.warning("the len(candidate_process_commodities) == 0.") - logger.info("TIP: Make sure the process names in the Processors_Candidate_Commodity_Data " - "are prefixed by the word 'candidate_'. e.g. 'candidate_HEFA'") - - return candidate_process_list, candidate_process_commodities - - -# ------------------------------------------------------------------------------ - - -def get_candidate_processor_slate_output_ratios(the_scenario, logger): - logger.info("start: get_candidate_processor_slate_output_ratios") - output_dict = {} - with sqlite3.connect(the_scenario.main_db) as db_con: - # first get the input commodities and quantities - sql = """ - select - cpl.process_id, - cpl.process_name, - cpc.io, - cpc.commodity_name, - cpc.commodity_id, - quantity, - units - from candidate_process_commodities cpc - join candidate_process_list cpl on cpl.process_id = cpc.process_id - where cpc.io = 'i' - ;""" - - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - - for row in db_data: - process_name = row[1] - io = row[2] - commodity_name = row[3] - commodity_id = row[4] - quantity = float(row[5]) - units = row[6] - - if process_name not in list(output_dict.keys()): - output_dict[process_name] = {} - output_dict[process_name]['i'] = [] - output_dict[process_name]['o'] = [] # initialize the output dict at the same time - output_dict[process_name]['i'].append([commodity_name, Q_(quantity, units)]) - - # next get the output commodities and quantities and scale them by the input quantities - # e.g. output scaled = output / input - sql = """ - select - cpl.process_id, - cpl.process_name, - cpc.io, - cpc.commodity_name, - cpc.commodity_id, - cpc.quantity, - cpc.units, - c.phase_of_matter - from candidate_process_commodities cpc - join candidate_process_list cpl on cpl.process_id = cpc.process_id - join commodities c on c.commodity_id = cpc.commodity_id - where cpc.io = 'o' - ;""" - - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - - for row in db_data: - process_name = row[1] - io = row[2] - commodity_name = row[3] - commodity_id = row[4] - quantity = float(row[5]) - units = row[6] - phase_of_matter = row[7] - - output_dict[process_name]['o'].append([commodity_name, Q_(quantity, units), phase_of_matter]) - - for process in output_dict: - if 1 != len(output_dict[process]['i']): - logger.warning("there is more than one input commodity specified for this process!!") - if 0 == len(output_dict[process]['i']): - logger.warning("there are no input commodities specified for this process!!") - if 0 == len(output_dict[process]['o']): - logger.warning("there are no output commodities specified for this process!!") - - return output_dict - - -# ============================================================================= - - -def processor_candidates(the_scenario, logger): - # ----------------------------------------------------------------------------- - - logger.info("start: generate_processor_candidates") - - # use candidate_nodes, candidate_process_list, - # and candidate_process_commodities to create the output - # product slate and candidate facility information including: - # (min_size, max_size, cost_formula) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - # clean-up candidate_processors table - # ------------------------------------ - logger.debug("drop the candidate_processors table") - main_db_con.execute("drop table if exists candidate_processors;") - main_db_con.commit() - - # create the candidate_processors table - # with the appropriate XY shape information from the nodeID - # --------------------------------------------------------- - logger.debug("create the candidate_processors table") - main_db_con.executescript( - """create table candidate_processors as - select - xy.shape_x shape_x, - xy.shape_y shape_y, - cpl.process_name || '_' || cn.node_id facility_name, - cpl.process_id process_id, - cpl.schedule_id schedule_id, - sn.schedule_name schedule_name, - cpc.commodity_name commodity_name, - cpc.commodity_id commodity_id, - cpl.maxsize quantity, - cpl.min_max_size_units units, - cpc.io io, - c.phase_of_matter phase_of_matter - from candidate_nodes cn - join candidate_process_commodities cpc on cpc.commodity_id = cn.commodity_id and cpc.process_id = cn.process_id - join candidate_process_list cpl on cpl.process_id = cn.process_id - join networkx_nodes xy on cn.node_id = xy.node_id - join commodities c on c.commodity_id = cpc.commodity_id - join schedule_names sn on sn.schedule_id = cpl.schedule_id - group by xy.shape_x, - xy.shape_y, - facility_name, - cpl.process_id, - cpl.schedule_id, - sn.schedule_name, - cpc.commodity_name, - cpc.commodity_id, - quantity, - cpc.units, - cpc.io, - c.phase_of_matter; - ;""") - main_db_con.commit() - - # generate the product slates for the candidate locations - # first get a dictionary of output commodity scalars per unit of input - - output_dict = get_candidate_processor_slate_output_ratios(the_scenario, logger) - - logger.info("opening a csv file") - with open(the_scenario.processor_candidates_commodity_data, 'w') as wf: - - # write the header line - header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io,schedule," \ - "max_processor_input" - wf.write(str(header_line + "\n")) - - # WRITE THE CSV FILE OF THE PROCESSOR CANDIDATES PRODUCT SLATE - - sql = """ - select - facility_name, 'processor', commodity_name, quantity, units, phase_of_matter, io, schedule_name, - cpl.process_name - from candidate_processors cp - join candidate_process_list cpl on cpl.process_id = cp.process_id - ;""" - db_cur = main_db_con.execute(sql) - db_data = db_cur.fetchall() - for row in db_data: - - facility_name = row[0] - facility_type = row[1] - commodity_name = row[2] - input_quantity = float(row[3]) - input_units = row[4] - phase_of_matter = row[5] - io = row[6] - schedule_name = row[7] - process_name = row[8] - max_processor_input = input_quantity - - wf.write("{},{},{},{},{},{},{},{},{}\n".format(row[0], row[1], row[2], row[3], row[4], row[5], row[6], - row[7], max_processor_input)) - - # write the scaled output commodities too - # first get the input for the denomenator - input_scaler_quantity = output_dict[process_name]['i'][0][1] - # then get the output scaler for the numerator - for output_scaler in output_dict[process_name]['o']: - output_commodity_name = output_scaler[0] - output_scaler_quantity = output_scaler[1] - output_phase_of_matter = output_scaler[2] - output_quantity = Q_(input_quantity, input_units) * output_scaler_quantity / input_scaler_quantity - wf.write( - "{},{},{},{},{},{},{},{},{}\n".format(row[0], row[1], output_commodity_name, output_quantity.magnitude, - output_quantity.units, output_phase_of_matter, 'o', - schedule_name, max_processor_input)) - - # MAKE THE FIRST PROCESSOR POINT LAYER - # this layer consists of candidate nodes where flow exceeds the min facility size at a RMP, - # or flow aggregates on the network above the min_facility size (anywhere it gets bigger) - # --------------------------------------------------------------------------------------------- - logger.info("create a feature class with all the candidate processor locations: all_candidate_processors") - scenario_gdb = the_scenario.main_gdb - all_candidate_processors_fc = os.path.join(scenario_gdb, "all_candidate_processors") - - if arcpy.Exists(all_candidate_processors_fc): - arcpy.Delete_management(all_candidate_processors_fc) - logger.debug("deleted existing {} layer".format(all_candidate_processors_fc)) - - arcpy.CreateFeatureclass_management(scenario_gdb, "all_candidate_processors", "POINT", "#", "DISABLED", "DISABLED", - ftot_supporting_gis.LCC_PROJ) - - # add fields and set capacity and prefunded fields. - # --------------------------------------------------------------------- - arcpy.AddField_management(all_candidate_processors_fc, "facility_name", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "candidate", "SHORT") - fields = ("SHAPE@X", "SHAPE@Y", "facility_name", "candidate") - icursor = arcpy.da.InsertCursor(all_candidate_processors_fc, fields) - - main_scenario_gdb = the_scenario.main_gdb - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - sql = """ - select shape_x, shape_y, facility_name - from candidate_processors - group by facility_name - ;""" - db_cur = main_db_con.execute(sql) - db_data = db_cur.fetchall() - - for candidate_processor in db_data: - shape_x = float(candidate_processor[0]) - shape_y = float(candidate_processor[1]) - facility_name = candidate_processor[2] - # offset slightly from the network node - offset_x = random.randrange(100, 250, 25) - offset_y = random.randrange(100, 250, 25) - shape_x += offset_x - shape_y += offset_y - - icursor.insertRow([shape_x, shape_y, facility_name, 1]) - - del icursor - - return - - -def generate_bulk_processor_candidates(the_scenario, logger): - logger.info("starting generate_bulk_processor_candidates") - - candidate_location_oids_dict = {} - commodities_max_transport_dist_dict = {} - - # the first routing will flow finished fuels. - # we can back track to the original commodity (or optionally assign a generic feedstock) - # so that we can aggregate feedstocks for candiate facilities locations. - - logger.debug( - "dropping then adding the optimal_feedstock_flows table with the feedstock_as_fuel flows on the route segments") - logger.debug( - "note: this table selects the flow from the first link in the route. this is because pulp sums all flows on " - "the link") - logger.warning( - "this can take a few minutes on the capacitated seattle network... might want to look into optimizing the sql " - "statement for larger runs") - sql1 = "DROP TABLE if exists optimal_feedstock_flows;" - - sql2 = """CREATE INDEX if not exists 'ors_index' ON 'optimal_route_segments' ( - 'scenario_rt_id', - 'rt_variant_id', - 'from_position' - );""" - - sql3 = """ - CREATE TABLE optimal_feedstock_flows as - select - ors.network_source_id as network_source_id, - ors.network_source_oid as network_source_oid, - odp.from_location_id as location_id, - ors.scenario_rt_id as scenario_rt_Id, - ors.rt_variant_id as rt_variant_id, - ors.from_position as from_position, - (select sum(cumm.miles) from optimal_route_segments cumm where (cumm.scenario_rt_id = - ors.scenario_rt_id and cumm.rt_variant_id = ors.rt_variant_id and ors.from_position >= - cumm.from_position )) as cumm_dist, - ors.commodity_name as feedstock_as_fuel_name, - (select orig_flow.commodity_flow from optimal_route_segments orig_flow where ( - orig_flow.scenario_rt_id = ors.scenario_rt_id)) as feedstock_as_fuel_flow , - null as commodity_name, - null as commodity_flow, - cast(null as real) as max_transport_distance, - null as ignore_link - from optimal_route_segments ors - join optimal_route_segments cumm on ( cumm.scenario_rt_id = ors.scenario_rt_id and ors.from_position - >= cumm.from_position ) - - join od_pairs odp on odp.scenario_rt_id = ors.scenario_rt_id - group by ors.scenario_rt_id, ors.rt_variant_id, ors.from_position - order by ors.scenario_rt_id, ors.rt_variant_id, ors.from_position - ;""" - - with sqlite3.connect(the_scenario.main_db) as db_con: - logger.debug("drop the optimal_feedstock_flows table") - db_con.execute(sql1) # drop the table - logger.debug("create the index on optimal_route_segments") - db_con.execute(sql2) # create the index on optimal_route_segments - logger.debug("create the optimal_feedstock_flows table and add the records") - db_con.execute(sql3) # create the table and add the records - - # now set the appropriate feedstock and quantities. - # then sum the flows by commodity - # then query the table for flows greater than the facility size but within - # the raw material transport distance - - logger.debug("creating a list of fuel ratios by location_id to scale the feedstock_as_fuel flows") - sql = """ - -- this is the code for selecting the rmps and rmps_as_proc facilities - -- along with the commodity quanities to get the ratio of feedstock - -- to feedstock as fuel - - select f.facility_name, fti.facility_type, c.commodity_name, c.max_transport_distance, fc.location_id, - fc.quantity, fc.units - from facility_commodities fc - join facilities f on f.facility_id = fc.facility_id - join commodities c on c.commodity_id = fc.commodity_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where fti.facility_type like 'raw_material_producer' - order by fc.location_id - ;""" - - with sqlite3.connect(the_scenario.main_db) as db_con: - db_cur = db_con.cursor() - db_cur.execute(sql) - - # iterate through the db and build the ratios - fuel_ratios = {} - for row in db_cur: - facility_name = row[0] - facility_type = row[1] - commodity_name = row[2] - max_transport_distance = row[3] - location_id = row[4] - quantity = row[5] - units = row[6] - if not location_id in list(fuel_ratios.keys()): - fuel_ratios[location_id] = [] - fuel_ratios[location_id].append(0) # one feedstock - fuel_ratios[location_id].append([]) # many feedstocks-as-fuel - - if not "_as_proc" in facility_name: - fuel_ratios[location_id][0] = ([commodity_name, Q_(quantity, units), max_transport_distance]) - else: - # this is the feedstock-as-fuel value - fuel_ratios[location_id][1].append([commodity_name, Q_(quantity, units)]) - - # now iterate through the table we just created to make a list of - # updated fields for a bulk update execute many - logger.debug("interating through the list of feedstock_as_fuel to generate the update_list for the execute many") - update_list = [] - sql = """ - select scenario_rt_id, rt_variant_id, location_id, feedstock_as_fuel_name, feedstock_as_fuel_flow - from optimal_feedstock_flows - """ - with sqlite3.connect(the_scenario.main_db) as db_con: - db_cur = db_con.cursor() - db_cur.execute(sql) - - # iterate through the db and build the ratios - for row in db_cur: - scenario_rt_id = row[0] - rt_variant_id = row[1] - location_id = row[2] - feedstock_as_fuel_name = row[3] - feedstock_as_fuel_flow = row[4] - - # now look up what feedstock came from that rmp_as_proc - feedstock_name = fuel_ratios[location_id][0][0] - feedstock_quant_and_units = fuel_ratios[location_id][0][1] - max_transport_distance = fuel_ratios[location_id][0][2] - - # iterate through the feedstock_as_fuels to scale the right records - for feedstock_as_fuel in fuel_ratios[location_id][1]: - ratio_feedstock_as_fuel_name = feedstock_as_fuel[0] - feedstock_as_fuel_quant_and_units = feedstock_as_fuel[1] - - if feedstock_as_fuel_name == ratio_feedstock_as_fuel_name: - # this is the commodity we want to scale: - feedstock_fuel_quant = float(feedstock_as_fuel_flow) * ( - feedstock_quant_and_units / feedstock_as_fuel_quant_and_units).magnitude - update_list.append( - [str(feedstock_name), str(feedstock_fuel_quant), float(max_transport_distance), scenario_rt_id, - rt_variant_id, location_id, float(max_transport_distance)]) - - logger.debug("starting the execute many to update the list of feedstocks and scaled quantities from the RMPs") - with sqlite3.connect(the_scenario.main_db) as db_con: - update_sql = """ - UPDATE optimal_feedstock_flows - set commodity_name = ?, commodity_flow = ?, max_transport_distance = ? - where scenario_rt_id = ? and rt_variant_id = ? and location_id = ? and cumm_dist <= ? - ;""" - db_con.executemany(update_sql, update_list) - db_con.commit() - - - sql = """drop table if exists candidates_aggregated_feedstock_flows;""" - db_con.execute(sql) - - sql = """create table candidates_aggregated_feedstock_flows as - select - ors.network_source_id as network_source_id, - ors.network_source_oid as network_source_oid, - commodity_name, - sum(commodity_flow) as cumm_comm_flow - from optimal_feedstock_flows ors - where ors.commodity_flow > 0 - group by ors.network_source_id, ors.network_source_oid, commodity_name; """ - db_con.execute(sql) - - # get the commodity name and quantity at each link that has flow - # >= minsize, and <= maxsize for that commodity - # store them in a dictionary keyed off the source_id - # need to match that to a product slate and scale the facility size - logger.info("opening a csv file") - with open(the_scenario.processor_candidates_commodity_data, 'w') as wf: - - # write the header line - header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" - wf.write(str(header_line + "\n")) - - candidate_location_oids_dict = {} - with sqlite3.connect(the_scenario.main_db) as db_con: - sql = """ - -- this query pulls the aggregated flows by commodity within - -- it joins in candidate processor information for that commodity - -- as specified in the facility_commodities table such as: - -- min_size, max_size, and process_type - -- note: that by convention the facility_name specified by the user - -- in the processor_candidate_slate.csv file is used to define the - -- process type. e.g. "candidate_hefa" - - select - caff.network_source_id, - caff.network_source_oid, - c.commodity_id, - caff.commodity_name, - caff.cumm_comm_flow, - c.units, - c.phase_of_matter, - f.facility_name process_type, - maxsize.quantity as max_size, - minsize.quantity as min_size - from candidates_aggregated_feedstock_flows caff - join commodities c on c.commodity_name = caff.commodity_name - join facility_commodities maxsize on maxsize.commodity_id = (select c2.commodity_id from commodities - c2 where c2.commodity_name = "maxsize") - join facility_commodities minsize on minsize.commodity_id = (select c3.commodity_id from commodities - c3 where c3.commodity_name = "minsize") - join facilities f on f.facility_id = minsize.facility_id - where caff.cumm_comm_flow < max_size and caff.cumm_comm_flow > min_size - ;""" - - db_cur = db_con.execute(sql) - - # key the candidate_location_oids_dict off the source_name, and the source_oid - # and append a list of feedstocks and flows for that link - i = 0 # counter to give the facilities unique names in the csv file - for row in db_cur: - i += 1 # increment the candidate generation index - network_source = row[0] - net_source_oid = row[1] - commodity_name = row[3] - commodity_flow = row[4] - commodity_units = row[5] - phase_of_matter = row[6] - processor_type = row[7] - - commodity_quantity_with_units = Q_(commodity_flow, commodity_units) - - ## WRITE THE CSV FILE OF THE PROCESSOR CANDIDATES PRODUCT SLATE - - # write the inputs from the crop object - facility_name = "{}_{}_{}".format(processor_type, commodity_name, i) - facility_type = "processor" - # phase of matter - io = "i" - logger.info("writing input commodity: {} and demand: {} \t {}".format(facility_name, commodity_flow, - commodity_units)) - wf.write("{},{},{},{},{},{},{}\n".format(facility_name, facility_type, commodity_name, commodity_flow, - commodity_units, phase_of_matter, io)) - - # write the outputs from the fuel_dict - # first get a dictionary of output commodities based on the amount of feedstock available. - # if the user specifies a processor_candidate_slate_data file - # use that to convert feedstock flows to scaled processor outputs - if the_scenario.processors_candidate_slate_data != 'None': - output_dict = ftot_supporting.make_rmp_as_proc_slate(the_scenario, commodity_name, - commodity_quantity_with_units, logger) - # otherwise try AFPAT - else: - # estimate max fuel from that commodity - max_conversion_process = "" - # values and gets the lowest kg/bbl of total fuel conversion rate (== max biomass -> fuel - # efficiency) - max_conversion_process = \ - ftot_supporting.get_max_fuel_conversion_process_for_commodity(commodity_name, the_scenario, - logger)[ - 0] - - output_dict = get_processor_slate(commodity_name, commodity_quantity_with_units, - max_conversion_process, the_scenario, logger) - - # write the outputs from the fuel_dict. - for ouput_commodity, values in iteritems(output_dict): - logger.info("processing commodity: {} quantity: {}".format(ouput_commodity, values[0])) - commodity = ouput_commodity - quantity = values[0] - value = quantity.magnitude - units = quantity.units - phase_of_matter = values[1] - io = "o" - - logger.info("writing outputs for: {} and commodity: {} \t {}".format(facility_name, value, units)) - wf.write("{},{},{},{},{},{},{}\n".format(facility_name, facility_type, commodity, value, units, - phase_of_matter, io)) - - # add the mode to the dictionary and initialize as empty dict - if network_source not in list(candidate_location_oids_dict.keys()): - candidate_location_oids_dict[network_source] = {} - - # add the network link to the dict and initialize as empty list - if not net_source_oid in list(candidate_location_oids_dict[network_source].keys()): - candidate_location_oids_dict[network_source][net_source_oid] = [] - - # prepare the candidate_location_oids_dict with all the information - # to "set" the candidate processors in the GIS. - candidate_location_oids_dict[network_source][net_source_oid].append([ \ - facility_name, # need this to match the CSV we just wrote - 0, - commodity_flow, - facility_name.replace("candidate_", ""), - "", # no secondary processing for now - "", # no tertiary processing for now - "", # feedstock_type - "", # source_category - commodity_name # feedstock_source; input commodity - ]) - # --------------------------------------------------------------------------------------------- - - logger.info( - "create a feature class with all the candidate processor locations: all_candidate_processors_at_segments") - scenario_gdb = the_scenario.main_gdb - all_candidate_processors_fc = os.path.join(scenario_gdb, "all_candidate_processors") - - if arcpy.Exists(all_candidate_processors_fc): - arcpy.Delete_management(all_candidate_processors_fc) - logger.debug("deleted existing {} layer".format(all_candidate_processors_fc)) - - arcpy.CreateFeatureclass_management(scenario_gdb, "all_candidate_processors", "POINT", "#", "DISABLED", "DISABLED", - ftot_supporting_gis.LCC_PROJ) - - # add fields and set capacity and prefunded fields. - # --------------------------------------------------------------------- - arcpy.AddField_management(all_candidate_processors_fc, "facility_name", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Prefunded", "SHORT") - arcpy.AddField_management(all_candidate_processors_fc, "Capacity", "DOUBLE") - arcpy.AddField_management(all_candidate_processors_fc, "Primary_Processing_Type", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Secondary_Processing_Type", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Tertiary_Processing_Type", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Feedstock_Type", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Source_Category", "TEXT") - arcpy.AddField_management(all_candidate_processors_fc, "Feedstock_Source", "TEXT") - fields = ("SHAPE@X", "SHAPE@Y", "facility_name", "Prefunded", "Capacity", "Primary_Processing_Type", - "Secondary_Processing_Type", "Tertiary_Processing_Type", "Feedstock_Type", "Source_Category", - "Feedstock_Source") - icursor = arcpy.da.InsertCursor(all_candidate_processors_fc, fields) - - main_scenario_gdb = the_scenario.main_gdb - - for source_name in candidate_location_oids_dict: - with arcpy.da.SearchCursor(os.path.join(main_scenario_gdb, source_name), ["OBJECTID", "SHAPE@"]) as cursor: - - for row in cursor: - - if row[0] in candidate_location_oids_dict[source_name]: - # might need to giggle the locations here? - for candidate in candidate_location_oids_dict[source_name][row[0]]: - facility_name = candidate[0] - prefunded = candidate[1] - commodity_flow = candidate[2] - facility_type = candidate[3] - sec_proc_type = candidate[4] - tert_proc_type = candidate[5] - feedstock_type = candidate[6] - source_category = candidate[7] - commodity_name = candidate[8] - # off-set the candidates from each other - point = row[1].firstPoint - shape_x = point.X - shape_y = point.Y - offset_x = random.randrange(100, 250, 25) - offset_y = random.randrange(100, 250, 25) - shape_x += offset_x - shape_y += offset_y - - icursor.insertRow([ \ - shape_x, - shape_y, - facility_name, - prefunded, - commodity_flow, - facility_type, - sec_proc_type, - tert_proc_type, - feedstock_type, - source_category, - commodity_name \ - ]) - - del icursor - - # - - return -# ============================================================================== -def make_flat_locationxy_flow_dict(the_scenario, logger): - # gets the optimal flow for all time periods for each xy by commodity - - logger.info("starting make_processor_candidates_db") - - # Parse the Problem for the Optimal Solution - # returns the following structure: - # [optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material] - optimal_route_flows = parse_optimal_solution_db(the_scenario, logger)[1] - optimal_location_id_flow = {} - optimal_commodity_max_transport_dist_list = {} - - # get location_XYs from location_ids in main.db so we can save in a format thats useful to the route_cache - # xy_location_id_dict[location_id] = (x,y) - from ftot_routing import get_xy_location_id_dict - xy_location_id_dict = get_xy_location_id_dict(the_scenario, logger) - - # flatten the optimal_route_flows by time period - # ------------------------------------------------- - for route_id, optimal_data_list in iteritems(optimal_route_flows): - for data in optimal_data_list: - # break out the data: - # e.g optimal_route_flows[5633] : ['275, 189', 1, u'2', 63.791322], ['275, 189', 2, u'2', 63.791322]] - location_id_pair = data[0] - commodity_id = data[2] - optimal_flow = data[3] - - # keep track of which commodities are optimal - if not commodity_id in optimal_commodity_max_transport_dist_list: - optimal_commodity_max_transport_dist_list[commodity_id] = 0 - - # convert the location_id to a location_xy - from_location_xy = xy_location_id_dict[location_id_pair.split(',')[0]] - to_location_xy = xy_location_id_dict[location_id_pair.split(', ')[1]] # note the extraspace - location_xy_pair = "[{},{}]".format(from_location_xy, to_location_xy) - - if not location_xy_pair in optimal_location_id_flow: - optimal_location_id_flow[location_xy_pair] = {} - if not commodity_id in location_xy_pair[location_id_pair]: - optimal_location_id_flow[location_xy_pair][commodity_id] = optimal_flow - - optimal_location_id_flow[location_xy_pair][commodity_id] += optimal_flow - - -# =============================================================================== -def get_commodity_max_transport_dist_list(the_scenario, logger): - # need commodities maximum transport distance from the main.db - # ------------------------------------------------------------- - - optimal_commodity_max_transport_dist_list = [] - scenario_db = the_scenario.main_db - with sqlite3.connect(scenario_db) as db_con: - logger.info("connected to the db") - sql = "select commodity_id, commodity_name, max_transportation_distance from commodities" - db_cur = db_con.execute(sql) - - for row in db_cur: - commodity_id = row[0] - commodity_name = row[1] - max_trans_dist = row[2] - - print ("found optimal commodity: {} (id: {}) has max raw trans distance: {}".format(commodity_name, - commodity_id, - max_trans_dist)) - optimal_commodity_max_transport_dist_list[commodity_id] = max_trans_dist - - return optimal_commodity_max_transport_dist_list - - -# =============================================================================== - - -# ============================================================================== -def assign_state_names_and_offset_processors(the_scenario, processor_canidates_fc, logger): - # Assign state names to the processors, e.g., "ND1" - # --------------------------------------------------------------- - logger.info("Starting - Assign state names to the processors") - - scenario_gdb = the_scenario.main_gdb - temp_join_output = os.path.join(scenario_gdb, "temp_state_name_and_candidates_join") - - stateFC = os.path.join(the_scenario.common_data_folder, "base_layers\state.shp") - - if arcpy.Exists(temp_join_output): - logger.debug("Deleting existing temp_join_output {}".format(temp_join_output)) - arcpy.Delete_management(temp_join_output) - - arcpy.CalculateField_management(processor_canidates_fc, "facility_name", "!OBJECTID!", "PYTHON_9.3") - arcpy.SpatialJoin_analysis(processor_canidates_fc, stateFC, temp_join_output, "JOIN_ONE_TO_ONE", "KEEP_ALL", "", - "WITHIN") - logger.info("spatial join of all candidates with state FIPS.") - - initial_counter = 1 - processor_state_dict = {} - state_processor_counter_dict = {} - logger.info("iterating through the processors in each state and incrementing the name...e.g B:FL1, B:FL2, etc") - - with arcpy.da.SearchCursor(temp_join_output, ["facility_name", "STFIPS"]) as cursor: - - for row in cursor: - - state_fips = row[1] - state_abb = ftot_supporting_gis.get_state_abb_from_state_fips(state_fips) - - if state_abb not in state_processor_counter_dict: - state_processor_counter_dict[state_abb] = initial_counter - else: - state_processor_counter_dict[state_abb] += 1 - - processor_state_dict[row[0]] = str(state_abb).replace(" ", "") + str( - state_processor_counter_dict[state_abb]) - - # the processor_state_dict is all we needed from that temp join FC - # so now we can delete it. - arcpy.Delete_management(temp_join_output) - logger.debug("deleted temp_join_output {} layer".format(temp_join_output)) - - # Offset processor candidate locations from the network - # --------------------------------------------------------------- - - logger.info("Offset processor candidates from the network") - - logger.debug("Iterating through the processor_candidates_fc and updating the names and offsetting the locations.") - with arcpy.da.UpdateCursor(processor_canidates_fc, ["facility_name", "SHAPE@X", "SHAPE@Y"]) as cursor: - - for row in cursor: - - if row[0] in processor_state_dict: - row[0] = row[0] + str(processor_state_dict[row[0]]) - row[1] += 100 - row[2] += 100 - cursor.updateRow(row) - - return - -# ======================================================================== - -def get_processor_fc_summary_statistics(the_scenario, candidates_fc, logger): - logger.info("starting: get_processor_fc_summary_statistics") - # set gdb - scenario_gdb = the_scenario.main_gdb - - processor_fc = candidates_fc - # --------------------------------------------------------------------------------- - - # clean up the table if it exists in the db - logger.debug("clean up the processor_candidates table if it exists") - main_db = the_scenario.main_db - - with sqlite3.connect(main_db) as db_con: - sql = "DROP TABLE IF EXISTS processor_candidates;" - db_con.execute(sql) - - # create an empty table with the processor candidate information - sql = """CREATE TABLE processor_candidates(facility_name TEXT, SHAPE_X TEXT, SHAPE_Y TEXT, - CAPACITY TEXT, PREFUNDED TEXT, IDW_Weighting TEXT, - Feedstock_Type TEXT, Source_Category TEXT, Feedstock_Source TEXT, - Primary_Processing_Type TEXT, Secondary_Processing_Type TEXT, Tertiary_Processing_Type TEXT);""" - db_con.execute(sql) - - query = "ignore IS NULL" # not interested in facilities that get ignored - fields = ["facility_name", "SHAPE@X", "SHAPE@Y", - "CAPACITY", "PREFUNDED", "IDW_Weighting", - "Feedstock_Type", "Source_Category", "Feedstock_Source", - "Primary_Processing_Type", "Secondary_Processing_Type", "Tertiary_Processing_Type"] - with sqlite3.connect(main_db) as db_con: - with arcpy.da.SearchCursor(processor_fc, fields, where_clause=query) as scursor: - for row in scursor: - sql = """insert into processor_candidates - (facility_name, SHAPE_X, SHAPE_Y, - CAPACITY, PREFUNDED, IDW_Weighting, - Feedstock_Type, Source_Category, Feedstock_Source, - Primary_Processing_Type, Secondary_Processing_Type, Tertiary_Processing_Type) - VALUES ('{}', '{}', '{}', - '{}', '{}', '{}', - '{}', '{}', '{}', - '{}', '{}', '{}') - ;""".format(row[0], row[1], row[2], - row[3], row[4], row[5], - row[6], row[7], row[8], - row[9], row[10], row[11]) - db_con.execute(sql) - - ###### OLD PROCESSOR FC SUMMARY STATISTICS ************ ########### - - # Create output table - candidate_processor_summary_by_processing_type_table = os.path.join(scenario_gdb, - "candidate_processor_summary_by_processing_type") - candidate_processor_summary_by_feedstock_type_table = os.path.join(scenario_gdb, - "candidate_processor_summary_by_feedstock_type") - - if arcpy.Exists(candidate_processor_summary_by_processing_type_table): - arcpy.Delete_management(candidate_processor_summary_by_processing_type_table) - logger.debug("deleted existing {} layer".format(candidate_processor_summary_by_processing_type_table)) - - if arcpy.Exists(candidate_processor_summary_by_feedstock_type_table): - arcpy.Delete_management(candidate_processor_summary_by_feedstock_type_table) - logger.debug("deleted existing {} layer".format(candidate_processor_summary_by_feedstock_type_table)) - - # Summary Statistics on those fields - logger.info("starting: Statistics_analysis for candidate_processor_summary_by_processing_type_table") - arcpy.Statistics_analysis(in_table=processor_fc, out_table=candidate_processor_summary_by_processing_type_table, - statistics_fields="ignore SUM; CAPACITY SUM; IDW_Weighting MEAN; Prefunded SUM", - case_field="Primary_Processing_Type") - - logger.info("starting: Statistics_analysis for candidate_processor_summary_by_feedstock_type_table") - arcpy.Statistics_analysis(in_table=processor_fc, out_table=candidate_processor_summary_by_feedstock_type_table, - statistics_fields="ignore SUM; CAPACITY SUM; IDW_Weighting MEAN; Prefunded SUM", - case_field="Feedstock_Type") - - summary_dict = {} - - for table in ["candidate_processor_summary_by_processing_type", "candidate_processor_summary_by_feedstock_type"]: - - full_path_to_table = os.path.join(scenario_gdb, table) - - with arcpy.da.SearchCursor(full_path_to_table, "*") as search_cursor: # * accesses all fields in searchCursor - - for row in search_cursor: - - if row[1] is not None: - - table_short_name = table.replace("candidate_processor_summary_by_", "") - - if table_short_name not in list(summary_dict.keys()): - summary_dict[table_short_name] = {} - - summary_field = row[1].upper() - - summary_dict[table_short_name][summary_field] = {} - summary_dict[table_short_name][summary_field]["frequency"] = row[2] - summary_dict[table_short_name][summary_field]["ignore"] = row[3] - - summary_dict[table_short_name][summary_field]["Sum_Capacity"] = row[4] - summary_dict[table_short_name][summary_field]["Avg_IDW_Weighting"] = row[5] - - summary_dict[table_short_name][summary_field]["Prefunded"] = row[6] - - for table_key in sorted(summary_dict.keys()): - - table_dict = summary_dict[table_key] - - for summary_field_key in sorted(table_dict.keys()): - - summary_field_dict = table_dict[summary_field_key] - - for metric_key in sorted(summary_field_dict.keys()): - - metric_sum = summary_field_dict[metric_key] - - if metric_sum is None: - - metric_sum = 0 # sometimes summary statistics hands back null values which can't be cast as a - # float. - - logger.result('{}_{}_{}: \t{:,.1f}'.format(table_key.upper(), summary_field_key.upper(), metric_key, - float(metric_sum))) - +# ------------------------------------------------------------------------------ +# Name: ftot_processor.py +# +# Purpose: generates candidates from optimized flow of feedstock to the +# max_raw_material_transport distance +# +# ------------------------------------------------------------------------------ + +import os +import random +import sqlite3 +import datetime +import arcpy +import ftot_supporting +import ftot_supporting_gis +import pdb +from ftot_facilities import get_commodity_id +from ftot_facilities import get_schedule_id +from ftot_pulp import parse_optimal_solution_db +from ftot import Q_ +from six import iteritems + + +# ============================================================================== + + +def clean_candidate_processor_tables(the_scenario, logger): + logger.debug("start: clean_candidate_processor_tables") + + # clean up the tables, then create them. + # ---------------------------------------- + with sqlite3.connect(the_scenario.main_db) as db_con: + db_con.executescript(""" + drop table if exists candidate_process_list; + drop table if exists candidate_process_commodities; + + create table candidate_process_list( + process_id INTEGER PRIMARY KEY, + process_name text, + minsize numeric, + maxsize numeric, + min_max_size_units text, + cost_formula numeric, + cost_formula_units text, + min_aggregation numeric, + schedule_id integer + ); + + + create table candidate_process_commodities( + process_id integer, + io text, + commodity_name text, + commodity_id integer, + quantity numeric, + units text, + phase_of_matter text + ); + """) + + return + + +# ------------------------------------------------------------------------------ + + +def generate_candidate_processor_tables(the_scenario, logger): + logger.info("start: generate_candidate_processor_tables") + # drops, and then creates 2 tables related to candidates: + # (1) candidate_process_list, which creates a unique ID for each process name, + # and records the respective min and max size, units, and cost formula and units. + + # clean up the tables, then create them. + # ---------------------------------------- + clean_candidate_processor_tables(the_scenario, logger) + + # if the user specifies a processor_candidate_slate_data file + # use that to populate the candidate_processor_list and + # candidate_processor_commodities tables. + # + # note: the processors_candidate_slate_data attribute is user specified slates for generic conversion. + # whereas the processors_commodity_data is FTOT generated for specific candidates + if the_scenario.processors_candidate_slate_data != 'None': + + # get the slate from the facility_commodities table + # for all facilities named "candidate%" + # (note the wild card, '%' in the search). + candidate_process_list, candidate_process_commodities = get_candidate_process_data(the_scenario, logger) + + # process the candidate_process_list first so we can assign a process_id + # the method returns a dictionary keyed off the process_name and has process_id as the value. + # ----------------------------------------------------------------------- + process_id_name_dict = populate_candidate_process_list_table(the_scenario, candidate_process_list, logger) + + # process the candidate_process_commodities list. + # ------------------------------------------------ + populate_candidate_process_commodities(the_scenario, candidate_process_commodities, process_id_name_dict, + logger) + + +# ------------------------------------------------------------------------------ + + +def populate_candidate_process_commodities(the_scenario, candidate_process_commodities, process_id_name_dict, logger): + logger.info("start: populate_candidate_process_commodities") + + logger.debug("number of process IDs: {} ".format(len(process_id_name_dict.keys()))) + logger.debug("number of commodity records: {}".format(len(candidate_process_commodities))) + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # iterate through the list and get the process_id and commodity_id + for commodity in candidate_process_commodities: + logger.debug("start: {} ".format(commodity[0])) + + # replace the process_name with process_id + process_name = commodity[0] + process_id = process_id_name_dict[process_name] + commodity[0] = process_id + + # reusing the ftot_facilities.get_commodity_id module. + # so we need to get the dataset into the proper format it is expecting + # to add the commodity if it doesn't exist in the db yet. + commodity_name = commodity[1] + commodity_quantity = commodity[3] + commodity_unit = commodity[4] + commodity_phase = commodity[5] + commodity_max_transport_dist = 'Null' + io = commodity[6] + shared_max_transport_distance = 'N' + processor_max_input = commodity[3] + # empty string for facility type and schedule id because fields are not used + commodity_data = ['', commodity_name, commodity_quantity, commodity_unit, commodity_phase, commodity_max_transport_dist, io, shared_max_transport_distance, processor_max_input, ''] + + # get commodity_id. (adds commodity if it doesn't exist) + commodity_id = get_commodity_id(the_scenario, db_con, commodity_data, logger) + commodity[2] = commodity_id + + # now do an execute many on the lists for the segments and route_segments table + sql = "insert into candidate_process_commodities (" \ + "process_id, commodity_name, commodity_id, quantity, units, phase_of_matter, io) " \ + "values (?, ?, ?, ?, ?, ?, ?);" + db_con.executemany(sql, candidate_process_commodities) + db_con.commit() + + +# ------------------------------------------------------------------------------ + + +def create_commodity_id_name_dict(the_scenario, logger): + logger.debug("start: create_commodity_id_name_dict") + commodity_id_name_dict = {} + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = "select commodity_name, commodity_id from commodities;" + db_cur = db_con.execute(sql) + data = db_cur.fetchall() + for row in data: + commodity_id_name_dict[row[0]] = row[1] + logger.debug("returning {} commodity id/name records".format(len(commodity_id_name_dict))) + return commodity_id_name_dict + + +# ------------------------------------------------------------------------------ + + +def populate_candidate_process_list_table(the_scenario, candidate_process_list, logger): + logger.debug("start: populate_candidate_process_list_table") + + candidate_process_list_data = [] + + with sqlite3.connect(the_scenario.main_db) as db_con: + for process in candidate_process_list: + min_size = '' + min_size_units = '' + max_size = '' + max_size_units = '' + cost_formula = '' + cost_formula_units = '' + min_aggregation = '' + min_aggregation_units = '' + schedule_name = '' + process_name = process + + for process_property in candidate_process_list[process]: + if 'schedule_name' == process_property[0]: + schedule_name = process_property[1] + schedule_id = get_schedule_id(the_scenario, db_con, schedule_name, logger) + if 'minsize' == process_property[0]: + min_size = process_property[1] + min_size_units = str(process_property[2]) + if 'maxsize' == process_property[0]: + max_size = process_property[1] + max_size_units = str(process_property[2]) + if 'cost_formula' == process_property[0]: + cost_formula = process_property[1] + cost_formula_units = str(process_property[2]) + if 'min_aggregation' == process_property[0]: + min_aggregation = process_property[1] + min_aggregation_units = str(process_property[2]) + + # do some checks to make sure things aren't weird. + # ------------------------------------------------- + if max_size_units != min_size_units: + logger.warning("the units for the max_size and min_size candidate process do not match!") + if max_size_units != min_aggregation_units: + logger.warning("the units for the max_size and min_aggregation candidate process do not match!") + if min_size == '': + logger.warning("the min_size is set to Null") + if max_size == '': + logger.warning("the max_size is set to Null") + if min_aggregation == '': + logger.warning("the min_aggregation was not specified by csv and has been set to 1/4 min_size") + min_aggregation = float(min_size)/4 + if cost_formula == '': + logger.warning("the cost_formula is set to Null") + if cost_formula_units == '': + logger.warning("the cost_formula_units is set to Null") + + # otherwise, build up a list to add to the sql database. + candidate_process_list_data.append( + [process_name, min_size, max_size, max_size_units, cost_formula, cost_formula_units, min_aggregation, schedule_id]) + + # now do an execute many on the lists for the segments and route_segments table + + sql = "insert into candidate_process_list " \ + "(process_name, minsize, maxsize, min_max_size_units, cost_formula, cost_formula_units, " \ + "min_aggregation, schedule_id) values (?, ?, ?, ?, ?, ?, ?, ?);" + db_con.executemany(sql, candidate_process_list_data) + db_con.commit() + + # now return a dictionary of process_id and process_names + process_id_name_dict = {} + sql = "select process_name, process_id from candidate_process_list;" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + for row in db_data: + process_id_name_dict[row[0]] = row[1] + logger.info("note: added {} candidate processes to the candidate_process_list table".format( + len(list(process_id_name_dict.keys())))) + logger.debug("ID || Process Name:") + logger.debug("-----------------------") + for process_name in process_id_name_dict: + logger.debug("{} || {}".format(process_id_name_dict[process_name], process_name)) + return process_id_name_dict + + +# ===================================================================================================================== + + +def get_candidate_process_data(the_scenario, logger): + logger.info("start: get_candidate_process_data") + + from ftot_facilities import load_facility_commodities_input_data + + candidate_process_data = load_facility_commodities_input_data(the_scenario, + the_scenario.processors_candidate_slate_data, logger) + candidate_process_commodities = [] + candidate_process_list = {} + + for facility_name, facility_commodity_list in iteritems(candidate_process_data): + for row in facility_commodity_list: + commodity_name = row[1] + commodity_id = None + quantity = row[2] + units = str(row[3]) + phase_of_matter = row[4] + io = row[6] + schedule_name = row[-1] + + # store the input and output commodities + # in the candidate_process_commodities (CPC) list . + if io == 'i' or io == 'o': + candidate_process_commodities.append( + [facility_name, commodity_name, commodity_id, quantity, units, phase_of_matter, io]) + + # store the facility size and cost information in the + # candidate_process_list (CPL). + else: + if facility_name not in list(candidate_process_list.keys()): + candidate_process_list[facility_name] = [] + # add schedule name to the candidate_process_list array for the facility + candidate_process_list[facility_name].append(["schedule_name", schedule_name]) + + if commodity_name == 'maxsize': + candidate_process_list[facility_name].append(["maxsize", quantity, units]) + elif commodity_name == 'minsize': + candidate_process_list[facility_name].append(["minsize", quantity, units]) + elif commodity_name == 'cost_formula': + candidate_process_list[facility_name].append(["cost_formula", quantity, units]) + elif commodity_name == 'min_aggregation': + candidate_process_list[facility_name].append(["min_aggregation", quantity, units]) + + # log a warning if nothing came back from the query + if len(candidate_process_list) == 0: + logger.warning("the len(candidate_process_list) == 0.") + logger.info("TIP: Make sure the process names in the Processors_Candidate_Commodity_Data " + "are prefixed by the word 'candidate_'. e.g. 'candidate_HEFA'") + + # log a warning if nothing came back from the query + if len(candidate_process_commodities) == 0: + logger.warning("the len(candidate_process_commodities) == 0.") + logger.info("TIP: Make sure the process names in the Processors_Candidate_Commodity_Data " + "are prefixed by the word 'candidate_'. e.g. 'candidate_HEFA'") + + return candidate_process_list, candidate_process_commodities + + +# ------------------------------------------------------------------------------ + + +def get_candidate_processor_slate_output_ratios(the_scenario, logger): + logger.info("start: get_candidate_processor_slate_output_ratios") + output_dict = {} + with sqlite3.connect(the_scenario.main_db) as db_con: + # first get the input commodities and quantities + sql = """ + select + cpl.process_id, + cpl.process_name, + cpc.io, + cpc.commodity_name, + cpc.commodity_id, + quantity, + units + from candidate_process_commodities cpc + join candidate_process_list cpl on cpl.process_id = cpc.process_id + where cpc.io = 'i' + ;""" + + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + + for row in db_data: + process_name = row[1] + io = row[2] + commodity_name = row[3] + commodity_id = row[4] + quantity = float(row[5]) + units = row[6] + + if process_name not in list(output_dict.keys()): + output_dict[process_name] = {} + output_dict[process_name]['i'] = [] + output_dict[process_name]['o'] = [] # initialize the output dict at the same time + output_dict[process_name]['i'].append([commodity_name, Q_(quantity, units)]) + + # next get the output commodities and quantities and scale them by the input quantities + # e.g. output scaled = output / input + sql = """ + select + cpl.process_id, + cpl.process_name, + cpc.io, + cpc.commodity_name, + cpc.commodity_id, + cpc.quantity, + cpc.units, + c.phase_of_matter + from candidate_process_commodities cpc + join candidate_process_list cpl on cpl.process_id = cpc.process_id + join commodities c on c.commodity_id = cpc.commodity_id + where cpc.io = 'o' + ;""" + + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + + for row in db_data: + process_name = row[1] + io = row[2] + commodity_name = row[3] + commodity_id = row[4] + quantity = float(row[5]) + units = row[6] + phase_of_matter = row[7] + + output_dict[process_name]['o'].append([commodity_name, Q_(quantity, units), phase_of_matter]) + + for process in output_dict: + if 1 != len(output_dict[process]['i']): + logger.warning("there is more than one input commodity specified for this process!!") + if 0 == len(output_dict[process]['i']): + logger.warning("there are no input commodities specified for this process!!") + if 0 == len(output_dict[process]['o']): + logger.warning("there are no output commodities specified for this process!!") + + return output_dict + + +# ============================================================================= + + +def processor_candidates(the_scenario, logger): + # ----------------------------------------------------------------------------- + + logger.info("start: generate_processor_candidates") + + # use candidate_nodes, candidate_process_list, + # and candidate_process_commodities to create the output + # product slate and candidate facility information including: + # (min_size, max_size, cost_formula) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + # clean-up candidate_processors table + # ------------------------------------ + logger.debug("drop the candidate_processors table") + main_db_con.execute("drop table if exists candidate_processors;") + main_db_con.commit() + + # create the candidate_processors table + # with the appropriate XY shape information from the nodeID + # --------------------------------------------------------- + logger.debug("create the candidate_processors table") + main_db_con.executescript( + """create table candidate_processors as + select + xy.shape_x shape_x, + xy.shape_y shape_y, + cpl.process_name || '_' || cn.node_id facility_name, + cpl.process_id process_id, + cpl.schedule_id schedule_id, + sn.schedule_name schedule_name, + cpc.commodity_name commodity_name, + cpc.commodity_id commodity_id, + cpl.maxsize quantity, + cpl.min_max_size_units units, + cpc.io io, + c.phase_of_matter phase_of_matter + from candidate_nodes cn + join candidate_process_commodities cpc on cpc.commodity_id = cn.commodity_id and cpc.process_id = cn.process_id + join candidate_process_list cpl on cpl.process_id = cn.process_id + join networkx_nodes xy on cn.node_id = xy.node_id + join commodities c on c.commodity_id = cpc.commodity_id + join schedule_names sn on sn.schedule_id = cpl.schedule_id + group by xy.shape_x, + xy.shape_y, + facility_name, + cpl.process_id, + cpl.schedule_id, + sn.schedule_name, + cpc.commodity_name, + cpc.commodity_id, + quantity, + cpc.units, + cpc.io, + c.phase_of_matter; + ;""") + main_db_con.commit() + + # generate the product slates for the candidate locations + # first get a dictionary of output commodity scalars per unit of input + + output_dict = get_candidate_processor_slate_output_ratios(the_scenario, logger) + + logger.info("opening a csv file") + with open(the_scenario.processor_candidates_commodity_data, 'w') as wf: + + # write the header line + header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io,schedule," \ + "max_processor_input" + wf.write(str(header_line + "\n")) + + # WRITE THE CSV FILE OF THE PROCESSOR CANDIDATES PRODUCT SLATE + + sql = """ + select + facility_name, 'processor', commodity_name, quantity, units, phase_of_matter, io, schedule_name, + cpl.process_name + from candidate_processors cp + join candidate_process_list cpl on cpl.process_id = cp.process_id + ;""" + db_cur = main_db_con.execute(sql) + db_data = db_cur.fetchall() + for row in db_data: + + facility_name = row[0] + facility_type = row[1] + commodity_name = row[2] + input_quantity = float(row[3]) + input_units = row[4] + phase_of_matter = row[5] + io = row[6] + schedule_name = row[7] + process_name = row[8] + max_processor_input = input_quantity + + wf.write("{},{},{},{},{},{},{},{},{}\n".format(row[0], row[1], row[2], row[3], row[4], row[5], row[6], + row[7], max_processor_input)) + + # write the scaled output commodities too + # first get the input for the denomenator + input_scaler_quantity = output_dict[process_name]['i'][0][1] + # then get the output scaler for the numerator + for output_scaler in output_dict[process_name]['o']: + output_commodity_name = output_scaler[0] + output_scaler_quantity = output_scaler[1] + output_phase_of_matter = output_scaler[2] + output_quantity = Q_(input_quantity, input_units) * output_scaler_quantity / input_scaler_quantity + wf.write( + "{},{},{},{},{},{},{},{},{}\n".format(row[0], row[1], output_commodity_name, output_quantity.magnitude, + output_quantity.units, output_phase_of_matter, 'o', + schedule_name, max_processor_input)) + + # MAKE THE FIRST PROCESSOR POINT LAYER + # this layer consists of candidate nodes where flow exceeds the min facility size at a RMP, + # or flow aggregates on the network above the min_facility size (anywhere it gets bigger) + # --------------------------------------------------------------------------------------------- + logger.info("create a feature class with all the candidate processor locations: all_candidate_processors") + scenario_gdb = the_scenario.main_gdb + all_candidate_processors_fc = os.path.join(scenario_gdb, "all_candidate_processors") + + if arcpy.Exists(all_candidate_processors_fc): + arcpy.Delete_management(all_candidate_processors_fc) + logger.debug("deleted existing {} layer".format(all_candidate_processors_fc)) + + arcpy.CreateFeatureclass_management(scenario_gdb, "all_candidate_processors", "POINT", "#", "DISABLED", "DISABLED", + ftot_supporting_gis.LCC_PROJ) + + # add fields and set capacity and prefunded fields. + # --------------------------------------------------------------------- + arcpy.AddField_management(all_candidate_processors_fc, "facility_name", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "candidate", "SHORT") + fields = ("SHAPE@X", "SHAPE@Y", "facility_name", "candidate") + icursor = arcpy.da.InsertCursor(all_candidate_processors_fc, fields) + + main_scenario_gdb = the_scenario.main_gdb + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + sql = """ + select shape_x, shape_y, facility_name + from candidate_processors + group by facility_name + ;""" + db_cur = main_db_con.execute(sql) + db_data = db_cur.fetchall() + + for candidate_processor in db_data: + shape_x = float(candidate_processor[0]) + shape_y = float(candidate_processor[1]) + facility_name = candidate_processor[2] + # offset slightly from the network node + offset_x = random.randrange(100, 250, 25) + offset_y = random.randrange(100, 250, 25) + shape_x += offset_x + shape_y += offset_y + + icursor.insertRow([shape_x, shape_y, facility_name, 1]) + + del icursor + + return + + +def generate_bulk_processor_candidates(the_scenario, logger): + logger.info("starting generate_bulk_processor_candidates") + + candidate_location_oids_dict = {} + commodities_max_transport_dist_dict = {} + + # the first routing will flow finished fuels. + # we can back track to the original commodity (or optionally assign a generic feedstock) + # so that we can aggregate feedstocks for candiate facilities locations. + + logger.debug( + "dropping then adding the optimal_feedstock_flows table with the feedstock_as_fuel flows on the route segments") + logger.debug( + "note: this table selects the flow from the first link in the route. this is because pulp sums all flows on " + "the link") + logger.warning( + "this can take a few minutes on the capacitated seattle network... might want to look into optimizing the sql " + "statement for larger runs") + sql1 = "DROP TABLE if exists optimal_feedstock_flows;" + + sql2 = """CREATE INDEX if not exists 'ors_index' ON 'optimal_route_segments' ( + 'scenario_rt_id', + 'rt_variant_id', + 'from_position' + );""" + + sql3 = """ + CREATE TABLE optimal_feedstock_flows as + select + ors.network_source_id as network_source_id, + ors.network_source_oid as network_source_oid, + odp.from_location_id as location_id, + ors.scenario_rt_id as scenario_rt_Id, + ors.rt_variant_id as rt_variant_id, + ors.from_position as from_position, + (select sum(cumm.miles) from optimal_route_segments cumm where (cumm.scenario_rt_id = + ors.scenario_rt_id and cumm.rt_variant_id = ors.rt_variant_id and ors.from_position >= + cumm.from_position )) as cumm_dist, + ors.commodity_name as feedstock_as_fuel_name, + (select orig_flow.commodity_flow from optimal_route_segments orig_flow where ( + orig_flow.scenario_rt_id = ors.scenario_rt_id)) as feedstock_as_fuel_flow , + null as commodity_name, + null as commodity_flow, + cast(null as real) as max_transport_distance, + null as ignore_link + from optimal_route_segments ors + join optimal_route_segments cumm on ( cumm.scenario_rt_id = ors.scenario_rt_id and ors.from_position + >= cumm.from_position ) + + join od_pairs odp on odp.scenario_rt_id = ors.scenario_rt_id + group by ors.scenario_rt_id, ors.rt_variant_id, ors.from_position + order by ors.scenario_rt_id, ors.rt_variant_id, ors.from_position + ;""" + + with sqlite3.connect(the_scenario.main_db) as db_con: + logger.debug("drop the optimal_feedstock_flows table") + db_con.execute(sql1) # drop the table + logger.debug("create the index on optimal_route_segments") + db_con.execute(sql2) # create the index on optimal_route_segments + logger.debug("create the optimal_feedstock_flows table and add the records") + db_con.execute(sql3) # create the table and add the records + + # now set the appropriate feedstock and quantities. + # then sum the flows by commodity + # then query the table for flows greater than the facility size but within + # the raw material transport distance + + logger.debug("creating a list of fuel ratios by location_id to scale the feedstock_as_fuel flows") + sql = """ + -- this is the code for selecting the rmps and rmps_as_proc facilities + -- along with the commodity quanities to get the ratio of feedstock + -- to feedstock as fuel + + select f.facility_name, fti.facility_type, c.commodity_name, c.max_transport_distance, fc.location_id, + fc.quantity, fc.units + from facility_commodities fc + join facilities f on f.facility_id = fc.facility_id + join commodities c on c.commodity_id = fc.commodity_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where fti.facility_type like 'raw_material_producer' + order by fc.location_id + ;""" + + with sqlite3.connect(the_scenario.main_db) as db_con: + db_cur = db_con.cursor() + db_cur.execute(sql) + + # iterate through the db and build the ratios + fuel_ratios = {} + for row in db_cur: + facility_name = row[0] + facility_type = row[1] + commodity_name = row[2] + max_transport_distance = row[3] + location_id = row[4] + quantity = row[5] + units = row[6] + if not location_id in list(fuel_ratios.keys()): + fuel_ratios[location_id] = [] + fuel_ratios[location_id].append(0) # one feedstock + fuel_ratios[location_id].append([]) # many feedstocks-as-fuel + + if not "_as_proc" in facility_name: + fuel_ratios[location_id][0] = ([commodity_name, Q_(quantity, units), max_transport_distance]) + else: + # this is the feedstock-as-fuel value + fuel_ratios[location_id][1].append([commodity_name, Q_(quantity, units)]) + + # now iterate through the table we just created to make a list of + # updated fields for a bulk update execute many + logger.debug("interating through the list of feedstock_as_fuel to generate the update_list for the execute many") + update_list = [] + sql = """ + select scenario_rt_id, rt_variant_id, location_id, feedstock_as_fuel_name, feedstock_as_fuel_flow + from optimal_feedstock_flows + """ + with sqlite3.connect(the_scenario.main_db) as db_con: + db_cur = db_con.cursor() + db_cur.execute(sql) + + # iterate through the db and build the ratios + for row in db_cur: + scenario_rt_id = row[0] + rt_variant_id = row[1] + location_id = row[2] + feedstock_as_fuel_name = row[3] + feedstock_as_fuel_flow = row[4] + + # now look up what feedstock came from that rmp_as_proc + feedstock_name = fuel_ratios[location_id][0][0] + feedstock_quant_and_units = fuel_ratios[location_id][0][1] + max_transport_distance = fuel_ratios[location_id][0][2] + + # iterate through the feedstock_as_fuels to scale the right records + for feedstock_as_fuel in fuel_ratios[location_id][1]: + ratio_feedstock_as_fuel_name = feedstock_as_fuel[0] + feedstock_as_fuel_quant_and_units = feedstock_as_fuel[1] + + if feedstock_as_fuel_name == ratio_feedstock_as_fuel_name: + # this is the commodity we want to scale: + feedstock_fuel_quant = float(feedstock_as_fuel_flow) * ( + feedstock_quant_and_units / feedstock_as_fuel_quant_and_units).magnitude + update_list.append( + [str(feedstock_name), str(feedstock_fuel_quant), float(max_transport_distance), scenario_rt_id, + rt_variant_id, location_id, float(max_transport_distance)]) + + logger.debug("starting the execute many to update the list of feedstocks and scaled quantities from the RMPs") + with sqlite3.connect(the_scenario.main_db) as db_con: + update_sql = """ + UPDATE optimal_feedstock_flows + set commodity_name = ?, commodity_flow = ?, max_transport_distance = ? + where scenario_rt_id = ? and rt_variant_id = ? and location_id = ? and cumm_dist <= ? + ;""" + db_con.executemany(update_sql, update_list) + db_con.commit() + + + sql = """drop table if exists candidates_aggregated_feedstock_flows;""" + db_con.execute(sql) + + sql = """create table candidates_aggregated_feedstock_flows as + select + ors.network_source_id as network_source_id, + ors.network_source_oid as network_source_oid, + commodity_name, + sum(commodity_flow) as cumm_comm_flow + from optimal_feedstock_flows ors + where ors.commodity_flow > 0 + group by ors.network_source_id, ors.network_source_oid, commodity_name; """ + db_con.execute(sql) + + # get the commodity name and quantity at each link that has flow + # >= minsize, and <= maxsize for that commodity + # store them in a dictionary keyed off the source_id + # need to match that to a product slate and scale the facility size + logger.info("opening a csv file") + with open(the_scenario.processor_candidates_commodity_data, 'w') as wf: + + # write the header line + header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" + wf.write(str(header_line + "\n")) + + candidate_location_oids_dict = {} + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """ + -- this query pulls the aggregated flows by commodity within + -- it joins in candidate processor information for that commodity + -- as specified in the facility_commodities table such as: + -- min_size, max_size, and process_type + -- note: that by convention the facility_name specified by the user + -- in the processor_candidate_slate.csv file is used to define the + -- process type. e.g. "candidate_hefa" + + select + caff.network_source_id, + caff.network_source_oid, + c.commodity_id, + caff.commodity_name, + caff.cumm_comm_flow, + c.units, + c.phase_of_matter, + f.facility_name process_type, + maxsize.quantity as max_size, + minsize.quantity as min_size + from candidates_aggregated_feedstock_flows caff + join commodities c on c.commodity_name = caff.commodity_name + join facility_commodities maxsize on maxsize.commodity_id = (select c2.commodity_id from commodities + c2 where c2.commodity_name = "maxsize") + join facility_commodities minsize on minsize.commodity_id = (select c3.commodity_id from commodities + c3 where c3.commodity_name = "minsize") + join facilities f on f.facility_id = minsize.facility_id + where caff.cumm_comm_flow < max_size and caff.cumm_comm_flow > min_size + ;""" + + db_cur = db_con.execute(sql) + + # key the candidate_location_oids_dict off the source_name, and the source_oid + # and append a list of feedstocks and flows for that link + i = 0 # counter to give the facilities unique names in the csv file + for row in db_cur: + i += 1 # increment the candidate generation index + network_source = row[0] + net_source_oid = row[1] + commodity_name = row[3] + commodity_flow = row[4] + commodity_units = row[5] + phase_of_matter = row[6] + processor_type = row[7] + + commodity_quantity_with_units = Q_(commodity_flow, commodity_units) + + ## WRITE THE CSV FILE OF THE PROCESSOR CANDIDATES PRODUCT SLATE + + # write the inputs from the crop object + facility_name = "{}_{}_{}".format(processor_type, commodity_name, i) + facility_type = "processor" + # phase of matter + io = "i" + logger.info("writing input commodity: {} and demand: {} \t {}".format(facility_name, commodity_flow, + commodity_units)) + wf.write("{},{},{},{},{},{},{}\n".format(facility_name, facility_type, commodity_name, commodity_flow, + commodity_units, phase_of_matter, io)) + + # write the outputs from the fuel_dict + # first get a dictionary of output commodities based on the amount of feedstock available. + # if the user specifies a processor_candidate_slate_data file + # use that to convert feedstock flows to scaled processor outputs + if the_scenario.processors_candidate_slate_data != 'None': + output_dict = ftot_supporting.make_rmp_as_proc_slate(the_scenario, commodity_name, + commodity_quantity_with_units, logger) + # otherwise try AFPAT + else: + # estimate max fuel from that commodity + max_conversion_process = "" + # values and gets the lowest kg/bbl of total fuel conversion rate (== max biomass -> fuel + # efficiency) + max_conversion_process = \ + ftot_supporting.get_max_fuel_conversion_process_for_commodity(commodity_name, the_scenario, + logger)[ + 0] + + output_dict = get_processor_slate(commodity_name, commodity_quantity_with_units, + max_conversion_process, the_scenario, logger) + + # write the outputs from the fuel_dict. + for ouput_commodity, values in iteritems(output_dict): + logger.info("processing commodity: {} quantity: {}".format(ouput_commodity, values[0])) + commodity = ouput_commodity + quantity = values[0] + value = quantity.magnitude + units = quantity.units + phase_of_matter = values[1] + io = "o" + + logger.info("writing outputs for: {} and commodity: {} \t {}".format(facility_name, value, units)) + wf.write("{},{},{},{},{},{},{}\n".format(facility_name, facility_type, commodity, value, units, + phase_of_matter, io)) + + # add the mode to the dictionary and initialize as empty dict + if network_source not in list(candidate_location_oids_dict.keys()): + candidate_location_oids_dict[network_source] = {} + + # add the network link to the dict and initialize as empty list + if not net_source_oid in list(candidate_location_oids_dict[network_source].keys()): + candidate_location_oids_dict[network_source][net_source_oid] = [] + + # prepare the candidate_location_oids_dict with all the information + # to "set" the candidate processors in the GIS. + candidate_location_oids_dict[network_source][net_source_oid].append([ \ + facility_name, # need this to match the CSV we just wrote + 0, + commodity_flow, + facility_name.replace("candidate_", ""), + "", # no secondary processing for now + "", # no tertiary processing for now + "", # feedstock_type + "", # source_category + commodity_name # feedstock_source; input commodity + ]) + # --------------------------------------------------------------------------------------------- + + logger.info( + "create a feature class with all the candidate processor locations: all_candidate_processors_at_segments") + scenario_gdb = the_scenario.main_gdb + all_candidate_processors_fc = os.path.join(scenario_gdb, "all_candidate_processors") + + if arcpy.Exists(all_candidate_processors_fc): + arcpy.Delete_management(all_candidate_processors_fc) + logger.debug("deleted existing {} layer".format(all_candidate_processors_fc)) + + arcpy.CreateFeatureclass_management(scenario_gdb, "all_candidate_processors", "POINT", "#", "DISABLED", "DISABLED", + ftot_supporting_gis.LCC_PROJ) + + # add fields and set capacity and prefunded fields. + # --------------------------------------------------------------------- + arcpy.AddField_management(all_candidate_processors_fc, "facility_name", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Prefunded", "SHORT") + arcpy.AddField_management(all_candidate_processors_fc, "Capacity", "DOUBLE") + arcpy.AddField_management(all_candidate_processors_fc, "Primary_Processing_Type", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Secondary_Processing_Type", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Tertiary_Processing_Type", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Feedstock_Type", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Source_Category", "TEXT") + arcpy.AddField_management(all_candidate_processors_fc, "Feedstock_Source", "TEXT") + fields = ("SHAPE@X", "SHAPE@Y", "facility_name", "Prefunded", "Capacity", "Primary_Processing_Type", + "Secondary_Processing_Type", "Tertiary_Processing_Type", "Feedstock_Type", "Source_Category", + "Feedstock_Source") + icursor = arcpy.da.InsertCursor(all_candidate_processors_fc, fields) + + main_scenario_gdb = the_scenario.main_gdb + + for source_name in candidate_location_oids_dict: + with arcpy.da.SearchCursor(os.path.join(main_scenario_gdb, source_name), ["OBJECTID", "SHAPE@"]) as cursor: + + for row in cursor: + + if row[0] in candidate_location_oids_dict[source_name]: + # might need to giggle the locations here? + for candidate in candidate_location_oids_dict[source_name][row[0]]: + facility_name = candidate[0] + prefunded = candidate[1] + commodity_flow = candidate[2] + facility_type = candidate[3] + sec_proc_type = candidate[4] + tert_proc_type = candidate[5] + feedstock_type = candidate[6] + source_category = candidate[7] + commodity_name = candidate[8] + # off-set the candidates from each other + point = row[1].firstPoint + shape_x = point.X + shape_y = point.Y + offset_x = random.randrange(100, 250, 25) + offset_y = random.randrange(100, 250, 25) + shape_x += offset_x + shape_y += offset_y + + icursor.insertRow([ \ + shape_x, + shape_y, + facility_name, + prefunded, + commodity_flow, + facility_type, + sec_proc_type, + tert_proc_type, + feedstock_type, + source_category, + commodity_name \ + ]) + + del icursor + + # + + return +# ============================================================================== +def make_flat_locationxy_flow_dict(the_scenario, logger): + # gets the optimal flow for all time periods for each xy by commodity + + logger.info("starting make_processor_candidates_db") + + # Parse the Problem for the Optimal Solution + # returns the following structure: + # [optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material] + optimal_route_flows = parse_optimal_solution_db(the_scenario, logger)[1] + optimal_location_id_flow = {} + optimal_commodity_max_transport_dist_list = {} + + # get location_XYs from location_ids in main.db so we can save in a format thats useful to the route_cache + # xy_location_id_dict[location_id] = (x,y) + from ftot_routing import get_xy_location_id_dict + xy_location_id_dict = get_xy_location_id_dict(the_scenario, logger) + + # flatten the optimal_route_flows by time period + # ------------------------------------------------- + for route_id, optimal_data_list in iteritems(optimal_route_flows): + for data in optimal_data_list: + # break out the data: + # e.g optimal_route_flows[5633] : ['275, 189', 1, u'2', 63.791322], ['275, 189', 2, u'2', 63.791322]] + location_id_pair = data[0] + commodity_id = data[2] + optimal_flow = data[3] + + # keep track of which commodities are optimal + if not commodity_id in optimal_commodity_max_transport_dist_list: + optimal_commodity_max_transport_dist_list[commodity_id] = 0 + + # convert the location_id to a location_xy + from_location_xy = xy_location_id_dict[location_id_pair.split(',')[0]] + to_location_xy = xy_location_id_dict[location_id_pair.split(', ')[1]] # note the extraspace + location_xy_pair = "[{},{}]".format(from_location_xy, to_location_xy) + + if not location_xy_pair in optimal_location_id_flow: + optimal_location_id_flow[location_xy_pair] = {} + if not commodity_id in location_xy_pair[location_id_pair]: + optimal_location_id_flow[location_xy_pair][commodity_id] = optimal_flow + + optimal_location_id_flow[location_xy_pair][commodity_id] += optimal_flow + + +# =============================================================================== +def get_commodity_max_transport_dist_list(the_scenario, logger): + # need commodities maximum transport distance from the main.db + # ------------------------------------------------------------- + + optimal_commodity_max_transport_dist_list = [] + scenario_db = the_scenario.main_db + with sqlite3.connect(scenario_db) as db_con: + logger.info("connected to the db") + sql = "select commodity_id, commodity_name, max_transportation_distance from commodities" + db_cur = db_con.execute(sql) + + for row in db_cur: + commodity_id = row[0] + commodity_name = row[1] + max_trans_dist = row[2] + + print ("found optimal commodity: {} (id: {}) has max raw trans distance: {}".format(commodity_name, + commodity_id, + max_trans_dist)) + optimal_commodity_max_transport_dist_list[commodity_id] = max_trans_dist + + return optimal_commodity_max_transport_dist_list + + +# =============================================================================== + + +# ============================================================================== +def assign_state_names_and_offset_processors(the_scenario, processor_canidates_fc, logger): + # Assign state names to the processors, e.g., "ND1" + # --------------------------------------------------------------- + logger.info("Starting - Assign state names to the processors") + + scenario_gdb = the_scenario.main_gdb + temp_join_output = os.path.join(scenario_gdb, "temp_state_name_and_candidates_join") + + stateFC = os.path.join(the_scenario.common_data_folder, "base_layers\state.shp") + + if arcpy.Exists(temp_join_output): + logger.debug("Deleting existing temp_join_output {}".format(temp_join_output)) + arcpy.Delete_management(temp_join_output) + + arcpy.CalculateField_management(processor_canidates_fc, "facility_name", "!OBJECTID!", "PYTHON_9.3") + arcpy.SpatialJoin_analysis(processor_canidates_fc, stateFC, temp_join_output, "JOIN_ONE_TO_ONE", "KEEP_ALL", "", + "WITHIN") + logger.info("spatial join of all candidates with state FIPS.") + + initial_counter = 1 + processor_state_dict = {} + state_processor_counter_dict = {} + logger.info("iterating through the processors in each state and incrementing the name...e.g B:FL1, B:FL2, etc") + + with arcpy.da.SearchCursor(temp_join_output, ["facility_name", "STFIPS"]) as cursor: + + for row in cursor: + + state_fips = row[1] + state_abb = ftot_supporting_gis.get_state_abb_from_state_fips(state_fips) + + if state_abb not in state_processor_counter_dict: + state_processor_counter_dict[state_abb] = initial_counter + else: + state_processor_counter_dict[state_abb] += 1 + + processor_state_dict[row[0]] = str(state_abb).replace(" ", "") + str( + state_processor_counter_dict[state_abb]) + + # the processor_state_dict is all we needed from that temp join FC + # so now we can delete it. + arcpy.Delete_management(temp_join_output) + logger.debug("deleted temp_join_output {} layer".format(temp_join_output)) + + # Offset processor candidate locations from the network + # --------------------------------------------------------------- + + logger.info("Offset processor candidates from the network") + + logger.debug("Iterating through the processor_candidates_fc and updating the names and offsetting the locations.") + with arcpy.da.UpdateCursor(processor_canidates_fc, ["facility_name", "SHAPE@X", "SHAPE@Y"]) as cursor: + + for row in cursor: + + if row[0] in processor_state_dict: + row[0] = row[0] + str(processor_state_dict[row[0]]) + row[1] += 100 + row[2] += 100 + cursor.updateRow(row) + + return + +# ======================================================================== + +def get_processor_fc_summary_statistics(the_scenario, candidates_fc, logger): + logger.info("starting: get_processor_fc_summary_statistics") + # set gdb + scenario_gdb = the_scenario.main_gdb + + processor_fc = candidates_fc + # --------------------------------------------------------------------------------- + + # clean up the table if it exists in the db + logger.debug("clean up the processor_candidates table if it exists") + main_db = the_scenario.main_db + + with sqlite3.connect(main_db) as db_con: + sql = "DROP TABLE IF EXISTS processor_candidates;" + db_con.execute(sql) + + # create an empty table with the processor candidate information + sql = """CREATE TABLE processor_candidates(facility_name TEXT, SHAPE_X TEXT, SHAPE_Y TEXT, + CAPACITY TEXT, PREFUNDED TEXT, IDW_Weighting TEXT, + Feedstock_Type TEXT, Source_Category TEXT, Feedstock_Source TEXT, + Primary_Processing_Type TEXT, Secondary_Processing_Type TEXT, Tertiary_Processing_Type TEXT);""" + db_con.execute(sql) + + query = "ignore IS NULL" # not interested in facilities that get ignored + fields = ["facility_name", "SHAPE@X", "SHAPE@Y", + "CAPACITY", "PREFUNDED", "IDW_Weighting", + "Feedstock_Type", "Source_Category", "Feedstock_Source", + "Primary_Processing_Type", "Secondary_Processing_Type", "Tertiary_Processing_Type"] + with sqlite3.connect(main_db) as db_con: + with arcpy.da.SearchCursor(processor_fc, fields, where_clause=query) as scursor: + for row in scursor: + sql = """insert into processor_candidates + (facility_name, SHAPE_X, SHAPE_Y, + CAPACITY, PREFUNDED, IDW_Weighting, + Feedstock_Type, Source_Category, Feedstock_Source, + Primary_Processing_Type, Secondary_Processing_Type, Tertiary_Processing_Type) + VALUES ('{}', '{}', '{}', + '{}', '{}', '{}', + '{}', '{}', '{}', + '{}', '{}', '{}') + ;""".format(row[0], row[1], row[2], + row[3], row[4], row[5], + row[6], row[7], row[8], + row[9], row[10], row[11]) + db_con.execute(sql) + + ###### OLD PROCESSOR FC SUMMARY STATISTICS ************ ########### + + # Create output table + candidate_processor_summary_by_processing_type_table = os.path.join(scenario_gdb, + "candidate_processor_summary_by_processing_type") + candidate_processor_summary_by_feedstock_type_table = os.path.join(scenario_gdb, + "candidate_processor_summary_by_feedstock_type") + + if arcpy.Exists(candidate_processor_summary_by_processing_type_table): + arcpy.Delete_management(candidate_processor_summary_by_processing_type_table) + logger.debug("deleted existing {} layer".format(candidate_processor_summary_by_processing_type_table)) + + if arcpy.Exists(candidate_processor_summary_by_feedstock_type_table): + arcpy.Delete_management(candidate_processor_summary_by_feedstock_type_table) + logger.debug("deleted existing {} layer".format(candidate_processor_summary_by_feedstock_type_table)) + + # Summary Statistics on those fields + logger.info("starting: Statistics_analysis for candidate_processor_summary_by_processing_type_table") + arcpy.Statistics_analysis(in_table=processor_fc, out_table=candidate_processor_summary_by_processing_type_table, + statistics_fields="ignore SUM; CAPACITY SUM; IDW_Weighting MEAN; Prefunded SUM", + case_field="Primary_Processing_Type") + + logger.info("starting: Statistics_analysis for candidate_processor_summary_by_feedstock_type_table") + arcpy.Statistics_analysis(in_table=processor_fc, out_table=candidate_processor_summary_by_feedstock_type_table, + statistics_fields="ignore SUM; CAPACITY SUM; IDW_Weighting MEAN; Prefunded SUM", + case_field="Feedstock_Type") + + summary_dict = {} + + for table in ["candidate_processor_summary_by_processing_type", "candidate_processor_summary_by_feedstock_type"]: + + full_path_to_table = os.path.join(scenario_gdb, table) + + with arcpy.da.SearchCursor(full_path_to_table, "*") as search_cursor: # * accesses all fields in searchCursor + + for row in search_cursor: + + if row[1] is not None: + + table_short_name = table.replace("candidate_processor_summary_by_", "") + + if table_short_name not in list(summary_dict.keys()): + summary_dict[table_short_name] = {} + + summary_field = row[1].upper() + + summary_dict[table_short_name][summary_field] = {} + summary_dict[table_short_name][summary_field]["frequency"] = row[2] + summary_dict[table_short_name][summary_field]["ignore"] = row[3] + + summary_dict[table_short_name][summary_field]["Sum_Capacity"] = row[4] + summary_dict[table_short_name][summary_field]["Avg_IDW_Weighting"] = row[5] + + summary_dict[table_short_name][summary_field]["Prefunded"] = row[6] + + for table_key in sorted(summary_dict.keys()): + + table_dict = summary_dict[table_key] + + for summary_field_key in sorted(table_dict.keys()): + + summary_field_dict = table_dict[summary_field_key] + + for metric_key in sorted(summary_field_dict.keys()): + + metric_sum = summary_field_dict[metric_key] + + if metric_sum is None: + + metric_sum = 0 # sometimes summary statistics hands back null values which can't be cast as a + # float. + + logger.result('{}_{}_{}: \t{:,.1f}'.format(table_key.upper(), summary_field_key.upper(), metric_key, + float(metric_sum))) + logger.info("finished: get_processor_fc_summary_statistics") \ No newline at end of file diff --git a/program/ftot_pulp.py b/program/ftot_pulp.py index e28127b..a350bb1 100644 --- a/program/ftot_pulp.py +++ b/program/ftot_pulp.py @@ -1,3679 +1,3669 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot_pulp -# -# Purpose: PulP optimization - create and run a modified facility location problem. -# Take NetworkX and GIS scenario data as recorded in main.db and convert to a structure of edges, nodes, vertices. -# Create variables for flow over edges, unmet demand, processor use, and candidate processors to build if present -# Solve cost minimization for unmet demand, transportation, and facility build costs -# Constraints ensure compliance with scenario requirements (e.g. max_route_capacity) -# as well as general problem structure (e.g. conservation_of_flow) -# --------------------------------------------------------------------------------------------------- - -import datetime -import pdb -import re -import sqlite3 -from collections import defaultdict -import os -from six import iteritems - -from pulp import * - -import ftot_supporting -from ftot_supporting import get_total_runtime_string -from ftot import Q_ - -# =================== constants============= -storage = 1 -primary = 0 -fixed_schedule_id = 2 -fixed_route_duration = 0 - -THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 - -storage_cost_1 = 0.01 -storage_cost_2 = 0.05 -facility_onsite_storage_max = 10000000000 -facility_onsite_storage_min = 0 - -default_max_capacity = 10000000000 -default_min_capacity = 0 - - -def o1(the_scenario, logger): - # create vertices, then edges for permitted modes, then set volume & capacity on edges - pre_setup_pulp(logger, the_scenario) - - -def o2(the_scenario, logger): - # create variables, problem to optimize, and constraints - prob = setup_pulp_problem(the_scenario, logger) - prob = solve_pulp_problem(prob, the_scenario, logger) - save_pulp_solution(the_scenario, prob, logger) - record_pulp_solution(the_scenario, logger) - from ftot_supporting import post_optimization - post_optimization(the_scenario, 'o2', logger) - - -# =============================================================================== - - -# helper function that reads in schedule data and returns dict of Schedule objects -def generate_schedules(the_scenario, logger): - logger.debug("start: generate_schedules") - default_availabilities = {} - day_availabilities = {} - last_day = 1 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - schedule_data = db_cur.execute(""" - select schedule_id, - day, - availability - - from schedules""") - - schedule_data = schedule_data.fetchall() - for row_a in schedule_data: - schedule_id = row_a[0] - day = row_a[1] - availability = float(row_a[2]) - - if day == 0: # denotes the default value - default_availabilities[schedule_id] = availability - elif schedule_id in day_availabilities.keys(): - day_availabilities[schedule_id][day] = availability - else: - day_availabilities[schedule_id] = {day: availability} # initialize sub-dic - # last_day is the greatest day specified across ALL schedules to avoid mismatch of length - if day > last_day: - last_day = day - - # make dictionary to store schedule objects - schedule_dict = {} - - # after reading in csv, parse data into dictionary object - for schedule_id in default_availabilities.keys(): - # initialize list of length - schedule_dict[schedule_id] = [default_availabilities[schedule_id] for i in range(last_day)] - # change different days to actual availability instead of the default values - # schedule_id may not be in day_availabilities if schedule has default value on all days - if schedule_id in day_availabilities.keys(): - for day in day_availabilities[schedule_id].keys(): - # schedule_dict[schedule - schedule_dict[schedule_id][day-1] = day_availabilities[schedule_id][day] - - for key in schedule_dict.keys(): - logger.debug("schedule name: " + str(key)) - logger.debug("availability: ") - logger.debug(schedule_dict[key]) - logger.debug("finished: generate_schedules") - - return schedule_dict, last_day - - -def make_vehicle_type_dict(the_scenario, logger): - - # check for vehicle type file - ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) - vehicle_types_path = os.path.join(ftot_program_directory, "lib", "vehicle_types.csv") - if not os.path.exists(vehicle_types_path): - logger.warning("warning: cannot find vehicle_types file: {}".format(vehicle_types_path)) - return {} # return empty dict - - # initialize vehicle property dict and read through vehicle_types CSV - vehicle_dict = {} - with open(vehicle_types_path, 'r') as vt: - line_num = 1 - for line in vt: - if line_num == 1: - pass # do nothing - else: - flds = line.rstrip('\n').split(',') - vehicle_label = flds[0] - mode = flds[1].lower() - vehicle_property = flds[2] - property_value = flds[3] - - # validate entries - assert vehicle_label not in ['Default', 'NA'], "Vehicle label: {} is a protected word. Please rename the vehicle.".format(vehicle_label) - - assert mode in ['road', 'water', 'rail'], "Mode: {} is not supported. Please specify road, water, or rail.".format(mode) - - assert vehicle_property in ['Truck_Load_Solid', 'Railcar_Load_Solid', 'Barge_Load_Solid', 'Truck_Load_Liquid', - 'Railcar_Load_Liquid', 'Barge_Load_Liquid', 'Pipeline_Crude_Load_Liquid', 'Pipeline_Prod_Load_Liquid', - 'Truck_Fuel_Efficiency', 'Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', - 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted', 'Barge_Fuel_Efficiency', - 'Barge_CO2_Emissions', 'Rail_Fuel_Efficiency', 'Railroad_CO2_Emissions'], \ - "Vehicle property: {} is not recognized. Refer to scenario.xml for supported property labels.".format(vehicle_property) - - # convert units - # Pint throws an exception if units are invalid - if vehicle_property in ['Truck_Load_Solid', 'Railcar_Load_Solid', 'Barge_Load_Solid']: - # convert csv value into default solid units - property_value = Q_(property_value).to(the_scenario.default_units_solid_phase) - elif vehicle_property in ['Truck_Load_Liquid', 'Railcar_Load_Liquid', 'Barge_Load_Liquid', 'Pipeline_Crude_Load_Liquid', 'Pipeline_Prod_Load_Liquid']: - # convert csv value into default liquid units - property_value = Q_(property_value).to(the_scenario.default_units_liquid_phase) - elif vehicle_property in ['Truck_Fuel_Efficiency', 'Barge_Fuel_Efficiency', 'Rail_Fuel_Efficiency']: - # convert csv value into miles per gallon - property_value = Q_(property_value).to('mi/gal') - elif vehicle_property in ['Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted']: - # convert csv value into grams per mile - property_value = Q_(property_value).to('g/mi') - elif vehicle_property in ['Barge_CO2_Emissions', 'Railroad_CO2_Emissions']: - # convert csv value into grams per default mass unit per mile - property_value = Q_(property_value).to('g/{}/mi'.format(the_scenario.default_units_solid_phase)) - else: - pass # do nothing - - # populate dictionary - if mode not in vehicle_dict: - # create entry for new mode type - vehicle_dict[mode] = {} - if vehicle_label not in vehicle_dict[mode]: - # create new vehicle key and add property - vehicle_dict[mode][vehicle_label] = {vehicle_property: property_value} - else: - # add property to existing vehicle - if vehicle_property in vehicle_dict[mode][vehicle_label].keys(): - logger.warning('Property: {} already exists for Vehicle: {}. Overwriting with value: {}'.\ - format(vehicle_property, vehicle_label, property_value)) - vehicle_dict[mode][vehicle_label][vehicle_property] = property_value - - line_num += 1 - - # ensure all properties are included - for mode in vehicle_dict: - if mode == 'road': - properties = ['Truck_Load_Solid', 'Truck_Load_Liquid', 'Truck_Fuel_Efficiency', - 'Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', - 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted'] - elif mode == 'water': - properties = ['Barge_Load_Solid', 'Barge_Load_Liquid', 'Barge_Fuel_Efficiency', - 'Barge_CO2_Emissions'] - elif mode == 'rail': - properties = ['Railcar_Load_Solid', 'Railcar_Load_Liquid', 'Rail_Fuel_Efficiency', - 'Railroad_CO2_Emissions'] - for vehicle_label in vehicle_dict[mode]: - for required_property in properties: - assert required_property in vehicle_dict[mode][vehicle_label].keys(), "Property: {} missing from Vehicle: {}".format(required_property, vehicle_label) - - return vehicle_dict - - -def vehicle_type_setup(the_scenario, logger): - - logger.info("START: vehicle_type_setup") - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - main_db_con.executescript(""" - drop table if exists vehicle_types; - create table vehicle_types( - mode text, - vehicle_label text, - property_name text, - property_value text, - CONSTRAINT unique_vehicle_and_property UNIQUE(mode, vehicle_label, property_name)) - ;""") - - vehicle_dict = make_vehicle_type_dict(the_scenario, logger) - for mode in vehicle_dict: - for vehicle_label in vehicle_dict[mode]: - for vehicle_property in vehicle_dict[mode][vehicle_label]: - - property_value = vehicle_dict[mode][vehicle_label][vehicle_property] - - main_db_con.execute(""" - insert or ignore into vehicle_types - (mode, vehicle_label, property_name, property_value) - VALUES - ('{}','{}','{}','{}') - ; - """.format(mode, vehicle_label, vehicle_property, property_value)) - - -def make_commodity_mode_dict(the_scenario, logger): - - logger.info("START: make_commodity_mode_dict") - - if the_scenario.commodity_mode_data == "None": - logger.info('commodity_mode_data file not specified.') - return {} # return empty dict - - # check if path to table exists - elif not os.path.exists(the_scenario.commodity_mode_data): - logger.warning("warning: cannot find commodity_mode_data file: {}".format(the_scenario.commodity_mode_data)) - return {} # return empty dict - - # initialize dict and read through commodity_mode CSV - commodity_mode_dict = {} - with open(the_scenario.commodity_mode_data, 'r') as rf: - line_num = 1 - header = None # will assign within for loop - for line in rf: - if line_num == 1: - header = line.rstrip('\n').split(',') - # Replace the short pipeline name with the long name - for h in range(len(header)): - if header[h] == 'pipeline_crude': - header[h] = 'pipeline_crude_trf_rts' - elif header[h] == 'pipeline_prod': - header[h] = 'pipeline_prod_trf_rts' - else: - flds = line.rstrip('\n').split(',') - commodity_name = flds[0].lower() - assignment = flds[1:] - if commodity_name in commodity_mode_dict.keys(): - logger.warning('Commodity: {} already exists. Overwriting with assignments: {}'.format(commodity_name, assignment)) - commodity_mode_dict[commodity_name] = dict(zip(header[1:], assignment)) - line_num += 1 - - # warn if trying to permit a mode that is not permitted in the scenario - for commodity in commodity_mode_dict: - for mode in commodity_mode_dict[commodity]: - if commodity_mode_dict[commodity][mode] != 'N' and mode not in the_scenario.permittedModes: - logger.warning("Mode: {} not permitted in scenario. Commodity: {} will not travel on this mode".format(mode, commodity)) - - return commodity_mode_dict - - -def commodity_mode_setup(the_scenario, logger): - - logger.info("START: commodity_mode_setup") - - # set up vehicle types table - vehicle_type_setup(the_scenario, logger) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - main_db_con.executescript(""" - drop table if exists commodity_mode; - - create table commodity_mode( - mode text, - commodity_id text, - commodity_phase text, - vehicle_label text, - allowed_yn text, - CONSTRAINT unique_commodity_and_mode UNIQUE(commodity_id, mode)) - ;""") - - # query commmodities table - commod = main_db_con.execute("select commodity_name, commodity_id, phase_of_matter from commodities where commodity_name <> 'multicommodity';") - commod = commod.fetchall() - commodities = {} - for row in commod: - commodity_name = row[0] - commodity_id = row[1] - phase_of_matter = row[2] - commodities[commodity_name] = (commodity_id, phase_of_matter) - - # query vehicle types table - vehs = main_db_con.execute("select distinct mode, vehicle_label from vehicle_types;") - vehs = vehs.fetchall() - vehicle_types = {} - for row in vehs: - mode = row[0] - vehicle_label = row[1] - if mode not in vehicle_types: - # add new mode to dictionary and start vehicle list - vehicle_types[mode] = [vehicle_label] - else: - # append new vehicle to mode's vehicle list - vehicle_types[mode].append(vehicle_label) - - # assign mode permissions and vehicle labels to commodities - commodity_mode_dict = make_commodity_mode_dict(the_scenario, logger) - logger.info("----- commodity/vehicle type table -----") - - for permitted_mode in the_scenario.permittedModes: - for k, v in iteritems(commodities): - commodity_name = k - commodity_id = v[0] - phase_of_matter = v[1] - - allowed = 'Y' # may be updated later in loop - vehicle_label = 'Default' # may be updated later in loop - - if commodity_name in commodity_mode_dict and permitted_mode in commodity_mode_dict[commodity_name]: - - # get user's entry for commodity and mode - assignment = commodity_mode_dict[commodity_name][permitted_mode] - - if assignment == 'Y': - if phase_of_matter != 'liquid' and 'pipeline' in permitted_mode: - # solids not permitted on pipeline. - # note that FTOT previously asserts no custom vehicle label is created for pipeline - logger.warning("commodity {} is not liquid and cannot travel through pipeline mode: {}".format( - commodity_name, permitted_mode)) - allowed = 'N' - vehicle_label = 'NA' - - elif assignment == 'N': - allowed = 'N' - vehicle_label = 'NA' - - else: - # user specified a vehicle type - allowed = 'Y' - if permitted_mode in vehicle_types and assignment in vehicle_types[permitted_mode]: - # accept user's assignment - vehicle_label = assignment - else: - # assignment not a known vehicle. fail. - raise Exception("improper vehicle label in Commodity_Mode_Data_csv for commodity: {}, mode: {}, and vehicle: {}". \ - format(commodity_name, permitted_mode, assignment)) - - elif 'pipeline' in permitted_mode: - # for unspecified commodities, default to not permitted on pipeline - allowed = 'N' - vehicle_label = 'NA' - - logger.info("Commodity name: {}, Mode: {}, Allowed: {}, Vehicle type: {}". \ - format(commodity_name, permitted_mode, allowed, vehicle_label)) - - # write table. only includes modes that are permitted in the scenario xml file. - main_db_con.execute(""" - insert or ignore into commodity_mode - (mode, commodity_id, commodity_phase, vehicle_label, allowed_yn) - VALUES - ('{}',{},'{}','{}','{}') - ; - """.format(permitted_mode, commodity_id, phase_of_matter, vehicle_label, allowed)) - - return - - -# =============================================================================== - - -def source_tracking_setup(the_scenario, logger): - logger.info("START: source_tracking_setup") - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - max_inputs = main_db_con.execute("""select max(inputs) from - (select count(fc.commodity_id) inputs - from facility_commodities fc, facility_type_id ft - where ft.facility_type = 'processor' - and fc.io = 'i' - group by fc.facility_id) - ;""") - - for row in max_inputs: - if row[0] == None: - max_inputs_to_a_processor = 0 - else: - max_inputs_to_a_processor = int(row[0]) - if max_inputs_to_a_processor > 1: - logger.warning("error, this version of the optimization step only functions correctly with a single input" - " commodity type per processor") - - main_db_con.executescript(""" - - insert or ignore into commodities(commodity_name) values ('multicommodity'); - - drop table if exists source_commodity_ref - ; - - create table source_commodity_ref(id INTEGER PRIMARY KEY, - source_facility_id integer, - source_facility_name text, - source_facility_type_id integer, --lets us differentiate spiderweb from processor - commodity_id integer, - commodity_name text, - units text, - phase_of_matter text, - max_transport_distance numeric, - max_transport_distance_flag text, - share_max_transport_distance text, - CONSTRAINT unique_source_and_name UNIQUE(commodity_id, source_facility_id)) - ; - - insert or ignore into source_commodity_ref ( - source_facility_id, - source_facility_name, - source_facility_type_id, - commodity_id, - commodity_name, - units, - phase_of_matter, - max_transport_distance, - max_transport_distance_flag, - share_max_transport_distance) - select - f.facility_id, - f.facility_name, - f.facility_type_id, - c.commodity_id, - c.commodity_name, - c.units, - c.phase_of_matter, - (case when c.max_transport_distance is not null then - c.max_transport_distance else Null end) max_transport_distance, - (case when c.max_transport_distance is not null then 'Y' else 'N' end) max_transport_distance_flag, - (case when ifnull(c.share_max_transport_distance, 'N') = 'Y' then 'Y' else 'N' end) share_max_transport_distance - - from commodities c, facilities f, facility_commodities fc - where f.facility_id = fc.facility_id - and f.ignore_facility = 'false' - and fc.commodity_id = c.commodity_id - and fc.io = 'o' - and ifnull(c.share_max_transport_distance, 'N') != 'Y' - ; - - insert or ignore into source_commodity_ref ( - source_facility_id, - source_facility_name, - source_facility_type_id, - commodity_id, - commodity_name, - units, - phase_of_matter, - max_transport_distance, - max_transport_distance_flag, - share_max_transport_distance) - select - sc.source_facility_id, - sc.source_facility_name, - sc.source_facility_type_id, - o.commodity_id, - c.commodity_name, - c.units, - c.phase_of_matter, - sc.max_transport_distance, - sc.max_transport_distance_flag, - o.share_max_transport_distance - from source_commodity_ref sc, facility_commodities i, facility_commodities o, commodities c - where o.share_max_transport_distance = 'Y' - and sc.commodity_id = i.commodity_id - and o.facility_id = i.facility_id - and o.io = 'o' - and i.io = 'i' - and o.commodity_id = c.commodity_id - ; - """ - ) - - return - - -# =============================================================================== - - -def generate_all_vertices(the_scenario, schedule_dict, schedule_length, logger): - logger.info("START: generate_all_vertices table") - - total_potential_production = {} - multi_commodity_name = "multicommodity" - - storage_availability = 1 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - logger.debug("create the vertices table") - # create the vertices table - main_db_con.executescript(""" - drop table if exists vertices - ; - - create table if not exists vertices ( - vertex_id INTEGER PRIMARY KEY, location_id, - facility_id integer, facility_name text, facility_type_id integer, schedule_day integer, - commodity_id integer, activity_level numeric, storage_vertex binary, - udp numeric, supply numeric, demand numeric, - source_facility_id integer, - iob text, --allows input, output, or both - CONSTRAINT unique_vertex UNIQUE(facility_id, schedule_day, commodity_id, source_facility_id, storage_vertex)) - ;""") - - # create indexes for the networkx nodes and links tables - logger.info("create an index for the networkx nodes and links tables") - main_db_con.executescript(""" - CREATE INDEX IF NOT EXISTS node_index ON networkx_nodes (node_id, location_id) - ; - - create index if not exists nx_edge_index on - networkx_edges(from_node_id, to_node_id, - artificial, mode_source, mode_source_OID, - miles, route_cost_scaling, capacity) - ; - """) - - # -------------------------------- - - db_cur = main_db_con.cursor() - # nested cursor - db_cur4 = main_db_con.cursor() - counter = 0 - total_facilities = 0 - - for row in db_cur.execute("select count(distinct facility_id) from facilities;"): - total_facilities = row[0] - - # create vertices for each non-ignored facility facility - # facility_type can be "raw_material_producer", "ultimate_destination","processor"; - # get id from facility_type_id table - # any other facility types are not currently handled - - facility_data = db_cur.execute(""" - select facility_id, - facility_type, - facility_name, - location_id, - f.facility_type_id, - schedule_id - - from facilities f, facility_type_id ft - where ignore_facility = '{}' - and f.facility_type_id = ft.facility_type_id; - """.format('false')) - facility_data = facility_data.fetchall() - for row_a in facility_data: - - db_cur2 = main_db_con.cursor() - facility_id = row_a[0] - facility_type = row_a[1] - facility_name = row_a[2] - facility_location_id = row_a[3] - facility_type_id = row_a[4] - schedule_id = row_a[5] - if counter % 10000 == 1: - logger.info("vertices created for {} facilities of {}".format(counter, total_facilities)) - for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): - logger.info('{} vertices created'.format(row_d[0])) - counter = counter + 1 - - if facility_type == "processor": - # actual processors - will deal with endcaps in edges section - - # create processor vertices for any commodities that do not inherit max transport distance - proc_data = db_cur2.execute("""select fc.commodity_id, - ifnull(fc.quantity, 0), - fc.units, - ifnull(c.supertype, c.commodity_name), - fc.io, - mc.commodity_id, - c.commodity_name, - ifnull(s.source_facility_id, 0) - from facility_commodities fc, commodities c, commodities mc - left outer join source_commodity_ref s - on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') - where fc.facility_id = {} - and fc.commodity_id = c.commodity_id - and mc.commodity_name = '{}';""".format(facility_id, multi_commodity_name)) - - proc_data = proc_data.fetchall() - # entry for each incoming commodity and its potential sources - # each outgoing commodity with this processor as their source IF there is a max commod distance - for row_b in proc_data: - - commodity_id = row_b[0] - quantity = row_b[1] - io = row_b[4] - id_for_mult_commodities = row_b[5] - commodity_name = row_b[6] - source_facility_id = row_b[7] - new_source_facility_id = facility_id - - # vertices for generic demand type, or any subtype specified by the destination - for day_before, availability in enumerate(schedule_dict[schedule_id]): - if io == 'i': - main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, - facility_type_id, facility_name, schedule_day, commodity_id, activity_level, - storage_vertex, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - '{}' );""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - id_for_mult_commodities, availability, primary, - new_source_facility_id, 'b')) - - main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, - facility_type_id, facility_name, schedule_day, commodity_id, activity_level, - storage_vertex, demand, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, {}, - {}, {}, {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, - facility_name, - day_before+1, commodity_id, storage_availability, storage, quantity, - source_facility_id, io)) - - else: - if commodity_name != 'total_fuel': - main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, - facility_type_id, facility_name, schedule_day, commodity_id, activity_level, - storage_vertex, supply, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, - {}, {}, {}, {}, '{}');""".format( - facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, storage_availability, storage, quantity, source_facility_id, io)) - - - elif facility_type == "raw_material_producer": - rmp_data = db_cur.execute("""select fc.commodity_id, fc.quantity, fc.units, - ifnull(s.source_facility_id, 0), io - from facility_commodities fc - left outer join source_commodity_ref s - on (fc.commodity_id = s.commodity_id - and s.max_transport_distance_flag = 'Y' - and s.source_facility_id = {}) - where fc.facility_id = {};""".format(facility_id, facility_id)) - - rmp_data = rmp_data.fetchall() - - for row_b in rmp_data: - commodity_id = row_b[0] - quantity = row_b[1] - # units = row_b[2] - source_facility_id = row_b[3] - iob = row_b[4] - - if commodity_id in total_potential_production: - total_potential_production[commodity_id] = total_potential_production[commodity_id] + quantity - else: - total_potential_production[commodity_id] = quantity - - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - activity_level, storage_vertex, supply, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, availability, primary, quantity, - source_facility_id, iob)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - activity_level, storage_vertex, supply, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, storage_availability, storage, quantity, - source_facility_id, iob)) - - elif facility_type == "storage": # storage facility - storage_fac_data = db_cur.execute("""select - fc.commodity_id, - fc.quantity, - fc.units, - ifnull(s.source_facility_id, 0), - io - from facility_commodities fc - left outer join source_commodity_ref s - on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') - where fc.facility_id = {} ;""".format(facility_id)) - - storage_fac_data = storage_fac_data.fetchall() - - for row_b in storage_fac_data: - commodity_id = row_b[0] - source_facility_id = row_b[3] # 0 if not source-tracked - iob = row_b[4] - - for day_before in range(schedule_length): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - storage_vertex, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, storage, - source_facility_id, iob)) - - elif facility_type == "ultimate_destination": - - dest_data = db_cur2.execute("""select - fc.commodity_id, - ifnull(fc.quantity, 0), - fc.units, - fc.commodity_id, - ifnull(c.supertype, c.commodity_name), - ifnull(s.source_facility_id, 0), - io - from facility_commodities fc, commodities c - left outer join source_commodity_ref s - on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') - where fc.facility_id = {} - and fc.commodity_id = c.commodity_id;""".format(facility_id)) - - dest_data = dest_data.fetchall() - - for row_b in dest_data: - commodity_id = row_b[0] - quantity = row_b[1] - commodity_supertype = row_b[4] - source_facility_id = row_b[5] - iob = row_b[6] - zero_source_facility_id = 0 # material merges at primary vertex - - # vertices for generic demand type, or any subtype specified by the destination - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - activity_level, storage_vertex, demand, udp, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, - {}, {}, {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, availability, primary, quantity, - the_scenario.unMetDemandPenalty, - zero_source_facility_id, iob)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - activity_level, storage_vertex, demand, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, storage_availability, storage, quantity, - source_facility_id, iob)) - # vertices for other fuel subtypes that match the destination's supertype - # if the subtype is in the commodity table, it is produced by some facility in the scenario - db_cur3 = main_db_con.cursor() - for row_c in db_cur3.execute("""select commodity_id, units from commodities - where supertype = '{}';""".format(commodity_supertype)): - new_commodity_id = row_c[0] - # new_units = row_c[1] - for day_before, availability in schedule_dict[schedule_id]: - main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, - facility_type_id, facility_name, schedule_day, commodity_id, activity_level, - storage_vertex, demand, udp, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, - {}, {}, {}, {}, {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, - facility_name, - day_before+1, new_commodity_id, availability, primary, - quantity, - the_scenario.unMetDemandPenalty, - zero_source_facility_id, iob)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, - activity_level, storage_vertex, demand, - source_facility_id, iob) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, - day_before+1, new_commodity_id, storage_availability, storage, quantity, - source_facility_id, iob)) - - else: - logger.warning( - "error, unexpected facility_type: {}, facility_type_id: {}".format(facility_type, facility_type_id)) - - for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): - logger.info('{} vertices created'.format(row_d[0])) - - logger.debug("total possible production in scenario: {}".format(total_potential_production)) - - -# =============================================================================== - - -def add_storage_routes(the_scenario, logger): - logger.info("start: add_storage_routes") - # these are loops to and from the same facility; when multiplied to edges, - # they will connect primary to storage vertices, and storage vertices day to day - # will always create edge for this route from storage to storage vertex - # IF a primary vertex exists, will also create an edge connecting the storage vertex to the primary - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - logger.debug("create the storage_routes table") - - main_db_con.execute("drop table if exists storage_routes;") - main_db_con.execute("""create table if not exists storage_routes as - select facility_name || '_storage' as route_name, - location_id, - facility_id, - facility_name as o_name, - facility_name as d_name, - {} as cost_1, - {} as cost_2, - 1 as travel_time, - {} as storage_max, - 0 as storage_min - from facilities - where ignore_facility = 'false' - ;""".format(storage_cost_1, storage_cost_2, facility_onsite_storage_max)) - - # drop and create route_reference table - # remove "drop" and replace with "create table if not exists" for cache - main_db_con.execute("drop table if exists route_reference;") - main_db_con.execute("""create table if not exists route_reference( - route_id INTEGER PRIMARY KEY, route_type text, route_name text, scenario_rt_id integer, from_node_id integer, - to_node_id integer, from_location_id integer, to_location_id integer, from_facility_id integer, to_facility_id integer, - commodity_id integer, phase_of_matter text, cost numeric, miles numeric, first_nx_edge_id integer, last_nx_edge_id integer, - CONSTRAINT unique_routes UNIQUE(route_type, route_name, scenario_rt_id));""") - main_db_con.execute( - "insert or ignore into route_reference(route_type, route_name, scenario_rt_id) select 'storage', route_name, 0 from storage" - "_routes;") - - return - - -# =============================================================================== - - -def generate_connector_and_storage_edges(the_scenario, logger): - logger.info("START: generate_connector_and_storage_edges") - - multi_commodity_name = "multicommodity" - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - if ('pipeline_crude_trf_rts' in the_scenario.permittedModes) or ( - 'pipeline_prod_trf_rts' in the_scenario.permittedModes): - logger.info("create indices for the capacity_nodes and pipeline_mapping tables") - main_db_con.executescript( - """ - CREATE INDEX IF NOT EXISTS pm_index - ON pipeline_mapping (id, id_field_name, mapping_id_field_name, mapping_id); - CREATE INDEX IF NOT EXISTS cn_index ON capacity_nodes (source, id_field_name, source_OID); - """) - - for row in db_cur.execute( - "select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): - id_for_mult_commodities = row[0] - - # create storage & connector edges - main_db_con.execute("drop table if exists edges;") - main_db_con.executescript(""" - create table edges (edge_id INTEGER PRIMARY KEY, - route_id integer, - from_node_id integer, - to_node_id integer, - start_day integer, - end_day integer, - commodity_id integer, - o_vertex_id integer, - d_vertex_id integer, - max_edge_capacity numeric, - volume numeric, - capac_minus_volume_zero_floor numeric, - min_edge_capacity numeric, - capacity_units text, - units_conversion_multiplier numeric, - edge_flow_cost numeric, - edge_flow_cost2 numeric, - edge_type text, - nx_edge_id integer, - mode text, - mode_oid integer, - miles numeric, - simple_mode text, - tariff_id numeric, - phase_of_matter text, - source_facility_id integer, - miles_travelled numeric, - children_created text, - edge_count_from_source integer, - total_route_cost numeric, - CONSTRAINT unique_nx_subc_day UNIQUE(nx_edge_id, commodity_id, source_facility_id, start_day)) - ; - - insert or ignore into edges (route_id, - start_day, end_day, - commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, - edge_flow_cost, edge_flow_cost2,edge_type, - source_facility_id) - select o.route_id, o.schedule_day, d.schedule_day, - o.commodity_id, o.vertex_id, d.vertex_id, - o.storage_max, o.storage_min, - o.cost_1, o.cost_2, 'storage', - o.source_facility_id - from vertices d, - (select v.vertex_id, v.schedule_day, - v.commodity_id, v.storage_vertex, v.source_facility_id, - t.* - from vertices v, - (select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, - storage_max, storage_min, rr.route_id, location_id, facility_id - from storage_routes sr, route_reference rr where sr.route_name = rr.route_name) t - where v.facility_id = t.facility_id - and v.storage_vertex = 1) o - where d.facility_id = o.facility_id - and d.schedule_day = o.schedule_day+o.travel_time - and d.commodity_id = o.commodity_id - and o.vertex_id != d.vertex_id - and d.storage_vertex = 1 - and d.source_facility_id = o.source_facility_id - ; - - insert or ignore into edges (route_id, start_day, end_day, - commodity_id, o_vertex_id, d_vertex_id, - edge_flow_cost, edge_type, - source_facility_id) - select s.route_id, s.schedule_day, p.schedule_day, - (case when s.commodity_id = {} then p.commodity_id else s.commodity_id end) commodity_id, - --inbound commodies start at storage and go into primary - --outbound starts at primary and goes into storage - --anything else is an error for a connector edge - (case when fc.io = 'i' then s.vertex_id - when fc.io = 'o' then p.vertex_id - else 0 end) as o_vertex, - (case when fc.io = 'i' then p.vertex_id - when fc.io = 'o' then s.vertex_id - else 0 end) as d_vertex, - 0, 'connector', - s.source_facility_id - from vertices p, facility_commodities fc, - --s for storage vertex info, p for primary vertex info - (select v.vertex_id, v.schedule_day, - v.commodity_id, v.storage_vertex, v.source_facility_id, - t.* - from vertices v, - (select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, - storage_max, storage_min, rr.route_id, location_id, facility_id - from storage_routes sr, route_reference rr where sr.route_name = rr.route_name) t --t is route data - where v.facility_id = t.facility_id - and v.storage_vertex = 1) s - --from storage into primary, same day = inbound connectors - where p.facility_id = s.facility_id - and p.schedule_day = s.schedule_day - and (p.commodity_id = s.commodity_id or p.commodity_id = {} ) - and p.facility_id = fc.facility_id - and fc.commodity_id = s.commodity_id - and p.storage_vertex = 0 - --either edge is inbound and aggregating, or kept separate by source, or primary vertex is not source tracked - and - (p.source_facility_id = 0 or p.source_facility_id = p.facility_id or p.source_facility_id = s.source_facility_id) - ;""".format(id_for_mult_commodities, id_for_mult_commodities)) - - for row_d in db_cur.execute("select count(distinct edge_id) from edges where edge_type = 'connector';"): - logger.info('{} connector edges created'.format(row_d[0])) - # clear any transport edges from table - db_cur.execute("delete from edges where edge_type = 'transport';") - - return - - -# =============================================================================== - - -def generate_first_edges_from_source_facilities(the_scenario, schedule_length, logger): - - logger.info("START: generate_first_edges_from_source_facilities") - # create edges table - # plan to generate start and end days based on nx edge time to traverse and schedule - # can still have route_id, but only for storage routes now; nullable - - # multi_commodity_name = "multicommodity" - transport_edges_created = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - edges_requiring_children = 0 - - counter = 0 - - # create transport edges, only between storage vertices and nodes, based on networkx graph - - commodity_mode_data = main_db_con.execute("select * from commodity_mode;") - commodity_mode_data = commodity_mode_data.fetchall() - commodity_mode_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - commodity_phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - commodity_mode_dict[mode, commodity_id] = allowed_yn - - - source_edge_data = main_db_con.execute("""select - ne.edge_id, - ifnull(CAST(fn.location_id as integer), 'NULL'), - ifnull(CAST(tn.location_id as integer), 'NULL'), - ne.mode_source, - ifnull(nec.phase_of_matter_id, 'NULL'), - nec.route_cost, - ne.from_node_id, - ne.to_node_id, - nec.dollar_cost, - ne.miles, - ne.capacity, - ne.artificial, - ne.mode_source_oid, - v.commodity_id, - v.schedule_day, - v.vertex_id, - v.source_facility_id, - tv.vertex_id, - ifnull(t.miles_travelled, 0), - ifnull(t.edge_count_from_source, 0), - t.mode - from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, vertices v, - facility_type_id ft, facility_commodities fc - left outer join (--get facilities with incoming transport edges with tracked mileage - select vi.facility_id, min(ei.miles_travelled) miles_travelled, ei.source_facility_id, - ei.edge_count_from_source, ei.mode - from edges ei, vertices vi - where vi.vertex_id = ei.d_vertex_id - and edge_type = 'transport' - and ifnull(miles_travelled, 0) > 0 - group by vi.facility_id, ei.source_facility_id, ei.mode) t - on t.facility_id = v.facility_id and t.source_facility_id = v.source_facility_id and ne.mode_source = t.mode - left outer join vertices tv - on (CAST(tn.location_id as integer) = tv.location_id and v.source_facility_id = tv.source_facility_id) - where v.location_id = CAST(fn.location_id as integer) - and fc.facility_id = v.facility_id - and fc.commodity_id = v.commodity_id - and fc.io = 'o' - and ft.facility_type_id = v.facility_type_id - and (ft.facility_type = 'raw_material_producer' or t.facility_id = v.facility_id) - and ne.from_node_id = fn.node_id - and ne.to_node_id = tn.node_id - and ne.edge_id = nec.edge_id - and ifnull(ne.capacity, 1) > 0 - and v.storage_vertex = 1 - and v.source_facility_id != 0 --max commodity distance applies - ;""") - source_edge_data = source_edge_data.fetchall() - for row_a in source_edge_data: - - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - # max_daily_capacity = row_a[10] - # artificial = row_a[11] - mode_oid = row_a[12] - commodity_id = row_a[13] - origin_day = row_a[14] - vertex_id = row_a[15] - source_facility_id = row_a[16] - to_vertex = row_a[17] - previous_miles_travelled = row_a[18] - previous_edge_count = row_a[19] - previous_mode = row_a[20] - - simple_mode = row_a[3].partition('_')[0] - edge_count_from_source = 1 + previous_edge_count - total_route_cost = route_cost - miles_travelled = previous_miles_travelled + miles - - if counter % 10000 == 0: - for row_d in db_cur.execute("select count(distinct edge_id) from edges;"): - logger.info('{} edges created'.format(row_d[0])) - counter = counter + 1 - - tariff_id = 0 - if simple_mode == 'pipeline': - - # find tariff_ids - - sql = "select mapping_id from pipeline_mapping " \ - "where id = {} and id_field_name = 'source_OID' " \ - "and source = '{}' and mapping_id is not null;".format( - mode_oid, mode) - for tariff_row in db_cur.execute(sql): - tariff_id = tariff_row[0] - - - if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys() \ - and commodity_mode_dict[mode, commodity_id] == 'Y': - - # Edges are placeholders for flow variables - # 4-17: if both ends have no location, iterate through viable commodities and days, create edge - # for all days (restrict by link schedule if called for) - # for all allowed commodities, as currently defined by link phase of matter - - # days range from 1 to schedule_length - if origin_day in range(1, schedule_length+1): - if origin_day + fixed_route_duration <= schedule_length: - # if link is traversable in the timeframe - if simple_mode != 'pipeline' or tariff_id >= 0: - # for allowed commodities - # step 1 from source is from non-Null location to (probably) null location - - if from_location != 'NULL' and to_location == 'NULL': - # for each day and commodity, - # get the corresponding origin vertex id to include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough - # than checking if facility type is 'ultimate destination' - # only connect to vertices with matching source_facility_id - # source_facility_id is zero for commodities without source tracking - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id,phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - miles_travelled, 'N', edge_count_from_source, total_route_cost)) - - - elif from_location != 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding origin and destination vertex ids - # to include with the edge info - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, {}, - {}, {}, {}, - '{}',{},'{}', {}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, to_vertex, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - miles_travelled, 'N', edge_count_from_source, total_route_cost)) - - for row in db_cur.execute("select count(distinct edge_id) from edges where edge_type = 'transport';"): - transport_edges_created = row[0] - logger.info('{} transport edges created'.format(transport_edges_created)) - for row in db_cur.execute("""select count(distinct edge_id), children_created from edges - where edge_type = 'transport' - group by children_created;"""): - if row[1] == 'N': - edges_requiring_children = row[0] - - elif row[1] == 'Y': - logger.info('{} transport edges that have already been checked for children'.format(row[0])) - edges_requiring_children = transport_edges_created - row[0] - # edges_requiring_children is updated under either condition here, since one may not be triggered - logger.info('{} transport edges that need children'.format(edges_requiring_children)) - - return - - -# =============================================================================== - - -def generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger): - # method only runs for commodities with a max commodity constraint - - logger.info("START: generate_all_edges_from_source_facilities") - - multi_commodity_name = "multicommodity" - # initializations - all of these get updated if >0 edges exist - edges_requiring_children = 0 - endcap_edges = 0 - edges_resolved = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - transport_edges_created = 0 - nx_edge_count = 0 - source_based_edges_created = 0 - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport' - and children_created in ('N', 'Y', 'E');"""): - source_based_edges_created = row_d[0] - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport';"""): - transport_edges_created = row_d[0] - nx_edge_count = row_d[1] - - - commodity_mode_data = main_db_con.execute("select * from commodity_mode;") - commodity_mode_data = commodity_mode_data.fetchall() - commodity_mode_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - commodity_phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - commodity_mode_dict[mode, commodity_id] = allowed_yn - - current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created - from edges - where edge_type = 'transport' - group by children_created - order by children_created asc;""") - current_edge_data = current_edge_data.fetchall() - for row in current_edge_data: - if row[1] == 'N': - edges_requiring_children = row[0] - elif row[1] == 'Y': - edges_resolved = row[0] - elif row[1] == 'E': - endcap_edges = row[0] - if source_based_edges_created == edges_resolved + endcap_edges: - edges_requiring_children = 0 - if source_based_edges_created == edges_requiring_children + endcap_edges: - edges_resolved = 0 - if source_based_edges_created == edges_requiring_children + edges_resolved: - endcap_edges = 0 - - logger.info( - '{} transport edges created; {} require children'.format(transport_edges_created, edges_requiring_children)) - - # set up a table to keep track of endcap nodes - sql = """ - drop table if exists endcap_nodes; - create table if not exists endcap_nodes( - node_id integer NOT NULL, - - location_id integer, - - --mode of edges that it's an endcap for - mode_source text NOT NULL, - - --facility it's an endcap for - source_facility_id integer NOT NULL, - - --commodities it's an endcap for - commodity_id integer NOT NULL, - - CONSTRAINT endcap_key PRIMARY KEY (node_id, mode_source, source_facility_id, commodity_id)) - --the combination of these four (excluding location_id) should be unique, - --and all fields except location_id should be filled - ;""" - db_cur.executescript(sql) - - # create transport edges, only between storage vertices and nodes, based on networkx graph - - edge_into_facility_counter = 0 - while_count = 0 - - while edges_requiring_children > 0: - while_count = while_count+1 - - # --get nx edges that align with the existing "in edges" - data from nx to create new edges --for each of - # those nx_edges, if they connect to more than one "in edge" in this batch, only consider connecting to - # the shortest -- if there is a valid nx_edge to build, the crossing node is not an endcap if total miles - # of the new route is over max transport distance, then endcap what if one child goes over max transport - # and another doesn't then the node will get flagged as an endcap, and another path may continue off it, - # allow both for now --check that day and commodity are permitted by nx - - potential_edge_data = main_db_con.execute(""" - select - ch.edge_id as ch_nx_edge_id, - ifnull(CAST(chfn.location_id as integer), 'NULL') fn_location_id, - ifnull(CAST(chtn.location_id as integer), 'NULL') tn_location_id, - ch.mode_source, - p.phase_of_matter, - nec.route_cost, - ch.from_node_id, - ch.to_node_id, - nec.dollar_cost, - ch.miles, - ch.capacity, - ch.artificial, - ch.mode_source_oid, - --parent edge into - p.commodity_id, - p.end_day, - --parent's dest. vertex if exists - ifnull(p.d_vertex_id,0) o_vertex, - p.source_facility_id, - p.leadin_miles_travelled, - - (p.edge_count_from_source +1) as new_edge_count, - (p.total_route_cost + nec.route_cost) new_total_route_cost, - p.edge_id leadin_edge, - p.nx_edge_id leadin_nx_edge, - - --new destination vertex if exists - ifnull(chtv.vertex_id,0) d_vertex, - - sc.max_transport_distance - - from - (select count(edge_id) parents, - min(miles_travelled) leadin_miles_travelled, - * - from edges - where children_created = 'N' - -----------------do not mess with this "group by" - group by to_node_id, source_facility_id, commodity_id, end_day - ------------------it affects which columns we're checking over for min miles travelled - --------------so that we only get the parent edges we want - order by parents desc - ) p, --parent edges to use in this batch - networkx_edges ch, - networkx_edge_costs nec, - source_commodity_ref sc, - networkx_nodes chfn, - networkx_nodes chtn - left outer join vertices chtv - on (CAST(chtn.location_id as integer) = chtv.location_id - and p.source_facility_id = chtv.source_facility_id - and chtv.commodity_id = p.commodity_id - and p.end_day = chtv.schedule_day - and chtv.iob = 'i') - - where p.to_node_id = ch.from_node_id - --and p.mode = ch.mode_source --build across modes, control at conservation of flow - and ch.to_node_id = chtn.node_id - and ch.from_node_id = chfn.node_id - and p.phase_of_matter = nec.phase_of_matter_id - and ch.edge_id = nec.edge_id - and ifnull(ch.capacity, 1) > 0 - and p.commodity_id = sc.commodity_id - ;""") - - # --should only get a single leadin edge per networkx/source/commodity/day combination - # leadin edge should be in route_data, current set of min. identifiers - # if we're trying to add an edge that has an entry in route_data, new miles travelled must be less - - potential_edge_data = potential_edge_data.fetchall() - - main_db_con.execute("update edges set children_created = 'Y' where children_created = 'N';") - - for row_a in potential_edge_data: - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - mode_oid = row_a[12] - commodity_id = row_a[13] - origin_day = row_a[14] - vertex_id = row_a[15] - source_facility_id = row_a[16] - leadin_edge_miles_travelled = row_a[17] - new_edge_count = row_a[18] - total_route_cost = row_a[19] - leadin_edge_id = row_a[20] - # leadin_nx_edge_id = row_a[21] - to_vertex = row_a[22] - max_commodity_travel_distance = row_a[23] - - # end_day = origin_day + fixed_route_duration - new_miles_travelled = miles + leadin_edge_miles_travelled - - if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys() \ - and commodity_mode_dict[mode, commodity_id] == 'Y': - - if new_miles_travelled > max_commodity_travel_distance: - # designate leadin edge as endcap - children_created = 'E' - # update the incoming edge to indicate it's an endcap - db_cur.execute( - "update edges set children_created = '{}' where edge_id = {}".format(children_created, - leadin_edge_id)) - if from_location != 'NULL': - db_cur.execute("""insert or ignore into endcap_nodes( - node_id, location_id, mode_source, source_facility_id, commodity_id) - VALUES ({}, {}, '{}', {}, {}); - """.format(from_node, from_location, mode, source_facility_id, commodity_id)) - else: - db_cur.execute("""insert or ignore into endcap_nodes( - node_id, mode_source, source_facility_id, commodity_id) - VALUES ({}, '{}', {}, {}); - """.format(from_node, mode, source_facility_id, commodity_id)) - - # create new edge - elif new_miles_travelled <= max_commodity_travel_distance: - - simple_mode = row_a[3].partition('_')[0] - tariff_id = 0 - if simple_mode == 'pipeline': - - # find tariff_ids - - sql = """select mapping_id - from pipeline_mapping - where id = {} - and id_field_name = 'source_OID' - and source = '{}' - and mapping_id is not null;""".format(mode_oid, mode) - for tariff_row in db_cur.execute(sql): - tariff_id = tariff_row[0] - - # if there are no edges yet for this day, nx, subc combination, - # AND this is the shortest existing leadin option for this day, nx, subc combination - # we'd be creating an edge for (otherwise wait for the shortest option) - # at this step, some leadin edge should always exist - - if origin_day in range(1, schedule_length+1): - if origin_day + fixed_route_duration <= schedule_length: - # if link is traversable in the timeframe - if simple_mode != 'pipeline' or tariff_id >= 0: - # for allowed commodities - - if from_location == 'NULL' and to_location == 'NULL': - # for each day and commodity, - # get the corresponding origin vertex id to include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough than - # checking if facility type is 'ultimate destination' - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id,phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, {}, {}, - '{}',{},'{}',{}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - # only create edge going into a location if an appropriate vertex exists - elif from_location == 'NULL' and to_location != 'NULL' and to_vertex > 0: - edge_into_facility_counter = edge_into_facility_counter + 1 - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}', {}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - to_vertex, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - # designate leadin edge as endcap - # this does, deliberately, allow endcap status to be - # overwritten if we've found a shorter path to a previous endcap - elif from_location != 'NULL' and to_location == 'NULL': - # for each day and commodity, get the corresponding origin vertex id - # to include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough than - # checking if facility type is 'ultimate destination' - # new for bsc, only connect to vertices with matching source facility id - # (only limited for RMP vertices) - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - - elif from_location != 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding origin and - # destination vertex ids to include with the edge info - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, - total_route_cost) - VALUES ({}, {}, - {}, {}, {}, - {}, {}, - {}, {}, {}, - '{}',{},'{}', {}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, to_vertex, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport' - and children_created in ('N', 'Y', 'E');"""): - source_based_edges_created = row_d[0] - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport';"""): - transport_edges_created = row_d[0] - nx_edge_count = row_d[1] - - current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created - from edges - where edge_type = 'transport' - group by children_created - order by children_created asc;""") - current_edge_data = current_edge_data.fetchall() - for row in current_edge_data: - if row[1] == 'N': - edges_requiring_children = row[0] - elif row[1] == 'Y': - edges_resolved = row[0] - elif row[1] == 'E': - endcap_edges = row[0] - logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) - if source_based_edges_created == edges_resolved + endcap_edges: - edges_requiring_children = 0 - if source_based_edges_created == edges_requiring_children + endcap_edges: - edges_resolved = 0 - if source_based_edges_created == edges_requiring_children + edges_resolved: - endcap_edges = 0 - - if while_count % 1000 == 0 or edges_requiring_children == 0: - logger.info( - '{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( - transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) - - # edges going in to the facility by re-running "generate first edges - # then re-run this method - - logger.info('{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( - transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) - logger.info("all source-based transport edges created") - - logger.info("create an index for the edges table by nodes") - - sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( - edge_id, route_id, from_node_id, to_node_id, commodity_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") - db_cur.execute(sql) - - return - - -# =============================================================================== - - -def generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger): - - global total_transport_routes - logger.info("START: generate_all_edges_without_max_commodity_constraint") - - multi_commodity_name = "multicommodity" - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - - commodity_mode_data = main_db_con.execute("select * from commodity_mode;") - commodity_mode_data = commodity_mode_data.fetchall() - commodity_mode_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - commodity_phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - commodity_mode_dict[mode, commodity_id] = allowed_yn - - counter = 0 - # for row in db_cur.execute( - # "select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): - # id_for_mult_commodities = row[0] - logger.info("COUNTING TOTAL TRANSPORT ROUTES") - for row in db_cur.execute(""" - select count(*) from networkx_edges, shortest_edges - WHERE networkx_edges.from_node_id = shortest_edges.from_node_id - AND networkx_edges.to_node_id = shortest_edges.to_node_id - AND networkx_edges.edge_id = shortest_edges.edge_id - ; - """): - total_transport_routes = row[0] - - # for all commodities with no max transport distance - source_facility_id = 0 - - # create transport edges, only between storage vertices and nodes, based on networkx graph - # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link - # iterate through nx edges: if neither node has a location, create 1 edge per viable commodity - # should also be per day, subject to nx edge schedule - # before creating an edge, check: commodity allowed by nx and max transport distance if not null - # will need nodes per day and commodity? or can I just check that with constraints? - # select data for transport edges - - sql = """select - ne.edge_id, - ifnull(fn.location_id, 'NULL'), - ifnull(tn.location_id, 'NULL'), - ne.mode_source, - ifnull(nec.phase_of_matter_id, 'NULL'), - nec.route_cost, - ne.from_node_id, - ne.to_node_id, - nec.dollar_cost, - ne.miles, - ne.capacity, - ne.artificial, - ne.mode_source_oid - from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, shortest_edges se - - where ne.from_node_id = fn.node_id - and ne.to_node_id = tn.node_id - and ne.edge_id = nec.edge_id - and ne.from_node_id = se.from_node_id -- match to the shortest_edges table - and ne.to_node_id = se.to_node_id - and ifnull(ne.capacity, 1) > 0 - ;""" - nx_edge_data = main_db_con.execute(sql) - nx_edge_data = nx_edge_data.fetchall() - for row_a in nx_edge_data: - - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - # max_daily_capacity = row_a[10] - mode_oid = row_a[12] - simple_mode = row_a[3].partition('_')[0] - - counter = counter + 1 - - tariff_id = 0 - if simple_mode == 'pipeline': - - # find tariff_ids - - sql = "select mapping_id from pipeline_mapping " \ - "where id = {} and id_field_name = 'source_OID' and source = '{}' " \ - "and mapping_id is not null;".format( - mode_oid, mode) - for tariff_row in db_cur.execute(sql): - tariff_id = tariff_row[0] - - - if mode in the_scenario.permittedModes: - - # Edges are placeholders for flow variables - # for all days (restrict by link schedule if called for) - # for all allowed commodities, as currently defined by link phase of matter - - for day in range(1, schedule_length+1): - if day + fixed_route_duration <= schedule_length: - # if link is traversable in the timeframe - if simple_mode != 'pipeline' or tariff_id >= 0: - # for allowed commodities that can be output by some facility in the scenario - for row_c in db_cur.execute("""select commodity_id - from source_commodity_ref s - where phase_of_matter = '{}' - and max_transport_distance_flag = 'N' - and share_max_transport_distance = 'N' - group by commodity_id, source_facility_id""".format(phase_of_matter)): - db_cur4 = main_db_con.cursor() - commodity_id = row_c[0] - # source_facility_id = row_c[1] # fixed to 0 for all edges created by this method - if (mode, commodity_id) in commodity_mode_dict.keys() \ - and commodity_mode_dict[mode, commodity_id] == 'Y': - if from_location == 'NULL' and to_location == 'NULL': - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - - elif from_location != 'NULL' and to_location == 'NULL': - # for each day and commodity, get the corresponding origin vertex id - # to include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough - # than checking if facility type is 'ultimate destination' - # new for bsc, only connect to vertices withr matching source_facility_id - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): - from_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - from_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - elif from_location == 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding destination vertex id - # to include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): - to_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - to_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - elif from_location != 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding origin and destination vertex - # ids to include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): - from_vertex_id = row_d[0] - db_cur5 = main_db_con.cursor() - for row_e in db_cur5.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): - to_vertex_id = row_e[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, - to_node_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, edge_type, - nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, {}, {}, {}, {}, {}, - {}, {}, {}, '{}',{},'{}', {},{},'{}',{},'{}',{} - )""".format(from_node, - to_node, day, - day + fixed_route_duration, - commodity_id, - from_vertex_id, - to_vertex_id, - default_min_capacity, - route_cost, - dollar_cost, - 'transport', - nx_edge_id, mode, - mode_oid, miles, - simple_mode, - tariff_id, - phase_of_matter, - source_facility_id)) - - logger.debug("all transport edges created") - - logger.info("all edges created") - logger.info("create an index for the edges table by nodes") - index_start_time = datetime.datetime.now() - sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( - edge_id, route_id, from_node_id, to_node_id, commodity_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") - db_cur.execute(sql) - logger.info("edge_index Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(index_start_time))) - return - - -# =============================================================================== - - -def generate_edges_from_routes(the_scenario, schedule_length, logger): - # ROUTES - create a transport edge for each route by commodity, day, etc. - logger.info("START: generate_edges_from_routes") - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - - # Filter edges table on those in shortest edges, group by - # phase, commodity, from, and to locations to populate a routes table - # db_cur.execute("""select distinct from_location_id, to_location_id, - # from shortest_edges se - # """) - - # From Olivia - db_cur.execute("""insert or ignore into route_reference (route_type,scenario_rt_id,from_node_id,to_node_id, - from_location_id,to_location_id,from_facility_id,to_facility_id,cost,miles,phase_of_matter,commodity_id,first_nx_edge_id,last_nx_edge_id) - select 'transport', odp.scenario_rt_id, odp.from_node_id, odp.to_node_id,odp.from_location_id,odp.to_location_id, - odp.from_facility_id, odp.to_facility_id, r2.cost, - r2.miles, odp.phase_of_matter, odp.commodity_id, r2.first_nx_edge, r2.last_nx_edge - FROM od_pairs odp, - (select r1.scenario_rt_id, r1.miles, r1.cost, r1.num_edges, re1.edge_id as first_nx_edge, re2.edge_id as last_nx_edge, r1.phase_of_matter from - (select scenario_rt_id, sum(e.miles) as miles, sum(e.route_cost) as cost, max(rt_order_ind) as num_edges, e.phase_of_matter_id as phase_of_matter --, count(e.edge_id) - from route_edges re - LEFT OUTER JOIN --everything from the route edges table, only edge data from the adhoc table that matches route_id - (select ne.edge_id, - nec.route_cost as route_cost, - ne.miles as miles, - ne.mode_source as mode, - nec.phase_of_matter_id - from networkx_edges ne, networkx_edge_costs nec --or Edges table? - where nec.edge_id = ne.edge_id) e --this is the adhoc edge info table - on re.edge_id=e.edge_id - group by scenario_rt_id, phase_of_matter) r1 - join (select * from route_edges where rt_order_ind = 1) re1 on r1.scenario_rt_id = re1.scenario_rt_id - join route_edges re2 on r1.scenario_rt_id = re2.scenario_rt_id and r1.num_edges = re2.rt_order_ind) r2 - where r2.scenario_rt_id = odp.scenario_rt_id and r2.phase_of_matter = odp.phase_of_matter - ;""") - - summary_route_data = main_db_con.execute("""select rr.route_id, f1.facility_name as from_facility, - f2.facility_name as to_facility, rr.cost, rr.miles - FROM route_reference rr - join facilities f1 on rr.from_facility_id = f1.facility_id - join facilities f2 on rr.to_facility_id = f2.facility_id; """) - # Print route data to file in debug folder - import csv - with open(os.path.join(the_scenario.scenario_run_directory, "debug", 'optimal_routes.csv'), 'w', newline='') as f: - writer = csv.writer(f) - writer.writerow(['route_id','from_facility','to_facility','routing_cost','miles']) - writer.writerows(summary_route_data) - - route_data = main_db_con.execute("select * from route_reference where route_type = 'transport';") - - # Add an edge for each route, (applicable) vertex, day, commodity - for row_a in route_data: - route_id = row_a[0] - from_node_id = row_a[4] - to_node_id = row_a[5] - from_location = row_a[6] - to_location = row_a[7] - commodity_id = row_a[10] - phase_of_matter = row_a[11] - cost = row_a[12] - miles = row_a[13] - source_facility_id = 0 - - for day in range(1, schedule_length+1): - if day + fixed_route_duration <= schedule_length: - # add edge from o_vertex to d_vertex - # for each day and commodity, get the corresponding origin and destination vertex - # ids to include with the edge info - db_cur4 = main_db_con.cursor() - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): - from_vertex_id = row_d[0] - db_cur5 = main_db_con.cursor() - for row_e in db_cur5.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} - and v.commodity_id = {} and v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): - to_vertex_id = row_e[0] - main_db_con.execute("""insert or ignore into edges (route_id, from_node_id, - to_node_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - edge_flow_cost, edge_type, - miles,phase_of_matter) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, - '{}', {},'{}' - )""".format(route_id, - from_node_id, - to_node_id, - day, - day + fixed_route_duration, - commodity_id, - from_vertex_id, to_vertex_id, - cost, - 'transport', - # nx_edge_id, mode, mode_oid, - miles, - # simple_mode, tariff_id, - phase_of_matter)) - #source_facility_id)) - - return - - -# =============================================================================== - - -def set_edges_volume_capacity(the_scenario, logger): - logger.info("starting set_edges_volume_capacity") - with sqlite3.connect(the_scenario.main_db) as main_db_con: - logger.debug("starting to record volume and capacity for non-pipeline edges") - - main_db_con.execute( - "update edges set volume = (select ifnull(ne.volume,0) from networkx_edges ne " - "where ne.edge_id = edges.nx_edge_id ) where simple_mode in ('rail','road','water');") - main_db_con.execute( - "update edges set max_edge_capacity = (select ne.capacity from networkx_edges ne " - "where ne.edge_id = edges.nx_edge_id) where simple_mode in ('rail','road','water');") - logger.debug("volume and capacity recorded for non-pipeline edges") - - logger.debug("starting to record volume and capacity for pipeline edges") - ## - main_db_con.executescript("""update edges set volume = - (select l.background_flow - from pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where edges.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(edges.mode, l.source)>0) - where simple_mode = 'pipeline' - ; - - update edges set max_edge_capacity = - (select l.capac - from pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where edges.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(edges.mode, l.source)>0) - where simple_mode = 'pipeline' - ;""") - logger.debug("volume and capacity recorded for pipeline edges") - logger.debug("starting to record units and conversion multiplier") - main_db_con.execute("""update edges - set capacity_units = - (case when simple_mode = 'pipeline' then 'kbarrels' - when simple_mode = 'road' then 'truckload' - when simple_mode = 'rail' then 'railcar' - when simple_mode = 'water' then 'barge' - else 'unexpected mode' end) - ;""") - main_db_con.execute("""update edges - set units_conversion_multiplier = - (case when simple_mode = 'pipeline' and phase_of_matter = 'liquid' then {} - when simple_mode = 'road' and phase_of_matter = 'liquid' then {} - when simple_mode = 'road' and phase_of_matter = 'solid' then {} - when simple_mode = 'rail' and phase_of_matter = 'liquid' then {} - when simple_mode = 'rail' and phase_of_matter = 'solid' then {} - when simple_mode = 'water' and phase_of_matter = 'liquid' then {} - when simple_mode = 'water' and phase_of_matter = 'solid' then {} - else 1 end) - ;""".format(THOUSAND_GALLONS_PER_THOUSAND_BARRELS, - the_scenario.truck_load_liquid.magnitude, - the_scenario.truck_load_solid.magnitude, - the_scenario.railcar_load_liquid.magnitude, - the_scenario.railcar_load_solid.magnitude, - the_scenario.barge_load_liquid.magnitude, - the_scenario.barge_load_solid.magnitude, - )) - logger.debug("units and conversion multiplier recorded for all edges; starting capacity minus volume") - main_db_con.execute("""update edges - set capac_minus_volume_zero_floor = - max((select (max_edge_capacity - ifnull(volume,0)) where max_edge_capacity is not null),0) - where max_edge_capacity is not null - ;""") - logger.debug("capacity minus volume (minimum set to zero) recorded for all edges") - return - - -# =============================================================================== - - -def pre_setup_pulp(logger, the_scenario): - logger.info("START: pre_setup_pulp") - - commodity_mode_setup(the_scenario, logger) - - # create table to track source facility of commodities with a max transport distance set - source_tracking_setup(the_scenario, logger) - - schedule_dict, schedule_length = generate_schedules(the_scenario, logger) - - generate_all_vertices(the_scenario, schedule_dict, schedule_length, logger) - - add_storage_routes(the_scenario, logger) - generate_connector_and_storage_edges(the_scenario, logger) - - if not the_scenario.ndrOn: - # start edges for commodities that inherit max transport distance - generate_first_edges_from_source_facilities(the_scenario, schedule_length, logger) - - # replicate all_routes by commodity and time into all_edges dictionary - generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger) - - # replicate all_routes by commodity and time into all_edges dictionary - generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger) - logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) - - else: - generate_edges_from_routes(the_scenario, schedule_length, logger) - - set_edges_volume_capacity(the_scenario, logger) - - return - - -# =============================================================================== - - -def create_flow_vars(the_scenario, logger): - logger.info("START: create_flow_vars") - - # we have a table called edges. - # call helper method to get list of unique IDs from the Edges table. - # use the rowid as a simple unique integer index - edge_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, source_facility_id - from edges;""") - edge_list_data = edge_list_cur.fetchall() - counter = 0 - for row in edge_list_data: - if counter % 500000 == 0: - logger.info( - "processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) - counter += 1 - # create an edge for each commodity allowed on this link - this construction may change - # as specific commodity restrictions are added - # running just with nodes for now, will add proper facility info and storage back soon - edge_list.append((row[0])) - - - flow_var = LpVariable.dicts("Edge", edge_list, 0, None) - return flow_var - - -# =============================================================================== - - -def create_unmet_demand_vars(the_scenario, logger): - logger.info("START: create_unmet_demand_vars") - demand_var_list = [] - # may create vertices with zero demand, but only for commodities that the facility has demand for at some point - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day, - ifnull(c.supertype, c.commodity_name) top_level_commodity_name, v.udp - from vertices v, commodities c, facility_type_id ft, facilities f - where v.commodity_id = c.commodity_id - and ft.facility_type = "ultimate_destination" - and v.storage_vertex = 0 - and v.facility_type_id = ft.facility_type_id - and v.facility_id = f.facility_id - and f.ignore_facility = 'false' - group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) - ;""".format('')): - # facility_id, day, and simplified commodity name - demand_var_list.append((row[0], row[1], row[2], row[3])) - - unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) - - return unmet_demand_var - - -# =============================================================================== - - -def create_candidate_processor_build_vars(the_scenario, logger): - logger.info("START: create_candidate_processor_build_vars") - processors_build_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute( - """select f.facility_id from facilities f, facility_type_id ft - where f.facility_type_id = ft.facility_type_id and facility_type = 'processor' - and candidate = 1 and ignore_facility = 'false' group by facility_id;"""): - # grab all candidate processor facility IDs - processors_build_list.append(row[0]) - - processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list, 0, None, 'Binary') - - return processor_build_var - - -# =============================================================================== - - -def create_binary_processor_vertex_flow_vars(the_scenario, logger): - logger.info("START: create_binary_processor_vertex_flow_vars") - processors_flow_var_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 0 - group by v.facility_id, v.schedule_day;"""): - # facility_id, day - processors_flow_var_list.append((row[0], row[1])) - - processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0, None, 'Binary') - - return processor_flow_var - - -# =============================================================================== - - -def create_processor_excess_output_vars(the_scenario, logger): - logger.info("START: create_processor_excess_output_vars") - - excess_var_list = [] - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - xs_cur = db_cur.execute(""" - select vertex_id, commodity_id - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 1;""") - # facility_id, day, and simplified commodity name - xs_data = xs_cur.fetchall() - for row in xs_data: - excess_var_list.append(row[0]) - - excess_var = LpVariable.dicts("XS", excess_var_list, 0, None) - - return excess_var - - -# =============================================================================== - - -def create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars): - logger.debug("START: create_opt_problem") - prob = LpProblem("Flow assignment", LpMinimize) - - unmet_demand_costs = [] - flow_costs = {} - processor_build_costs = [] - for u in unmet_demand_vars: - # facility_id = u[0] - # schedule_day = u[1] - # demand_commodity_name = u[2] - udp = u[3] - unmet_demand_costs.append(udp * unmet_demand_vars[u]) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - # Flow cost memory improvements: only get needed data; dict instead of list; narrow in lpsum - flow_cost_var = db_cur.execute("select edge_id, edge_flow_cost from edges e group by edge_id;") - flow_cost_data = flow_cost_var.fetchall() - counter = 0 - for row in flow_cost_data: - edge_id = row[0] - edge_flow_cost = row[1] - counter += 1 - - # flow costs cover transportation and storage - flow_costs[edge_id] = edge_flow_cost - # flow_costs.append(edge_flow_cost * flow_vars[(edge_id)]) - - logger.info("check if candidate tables exist") - sql = "SELECT name FROM sqlite_master WHERE type='table' " \ - "AND name in ('candidate_processors', 'candidate_process_list');" - count = len(db_cur.execute(sql).fetchall()) - - if count == 2: - - processor_build_cost = db_cur.execute(""" - select f.facility_id, (p.cost_formula*c.quantity) build_cost - from facilities f, facility_type_id ft, candidate_processors c, candidate_process_list p - where f.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and candidate = 1 - and ignore_facility = 'false' - and f.facility_name = c.facility_name - and c.process_id = p.process_id - group by f.facility_id, build_cost;""") - processor_build_cost_data = processor_build_cost.fetchall() - for row in processor_build_cost_data: - candidate_proc_facility_id = row[0] - proc_facility_build_cost = row[1] - processor_build_costs.append( - proc_facility_build_cost * processor_build_vars[candidate_proc_facility_id]) - - prob += (lpSum(unmet_demand_costs) + lpSum(flow_costs[k] * flow_vars[k] for k in flow_costs) + lpSum( - processor_build_costs)), "Total Cost of Transport, storage, facility building, and penalties" - - logger.debug("FINISHED: create_opt_problem") - return prob - - -# =============================================================================== - - -def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): - logger.debug("START: create_constraint_unmet_demand") - - # apply activity_level to get corresponding actual demand for var - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - # var has form(facility_name, day, simple_fuel) - # unmet demand commodity should be simple_fuel = supertype - - demand_met_dict = defaultdict(list) - actual_demand_dict = {} - - # demand_met = [] - # want to specify that all edges leading into this vertex + unmet demand = total demand - # demand primary (non-storage) vertices - - db_cur = main_db_con.cursor() - # each row_a is a primary vertex whose edges in contributes to the met demand of var - # will have one row for each fuel subtype in the scenario - unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - from vertices v, commodities c, facility_type_id ft, facilities f, edges e - where v.facility_id = f.facility_id - and ft.facility_type = 'ultimate_destination' - and f.facility_type_id = ft.facility_type_id - and f.ignore_facility = 'false' - and v.facility_type_id = ft.facility_type_id - and v.storage_vertex = 0 - and c.commodity_id = v.commodity_id - and e.d_vertex_id = v.vertex_id - group by v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - ;""") - - unmet_data = unmet_data.fetchall() - for row_a in unmet_data: - # primary_vertex_id = row_a[0] - # commodity_id = row_a[1] - var_full_demand = row_a[2] - proportion_of_supertype = row_a[3] - var_activity_level = row_a[4] - # source_facility_id = row_a[5] - facility_id = row_a[6] - day = row_a[7] - top_level_commodity = row_a[8] - udp = row_a[9] - edge_id = row_a[10] - var_actual_demand = var_full_demand * var_activity_level - - # next get inbound edges, apply appropriate modifier proportion to get how much of var's demand they satisfy - demand_met_dict[(facility_id, day, top_level_commodity, udp)].append( - flow_var[edge_id] * proportion_of_supertype) - actual_demand_dict[(facility_id, day, top_level_commodity, udp)] = var_actual_demand - - for key in unmet_demand_var: - if key in demand_met_dict: - # then there are some edges in - prob += lpSum(demand_met_dict[key]) == actual_demand_dict[key] - unmet_demand_var[ - key], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(key[0], - key[1], - key[2]) - else: - if key not in actual_demand_dict: - pdb.set_trace() - # no edges in, so unmet demand equals full demand - prob += actual_demand_dict[key] == unmet_demand_var[ - key], "constraint set unmet demand variable for facility {}, day {}, " \ - "commodity {} - no edges able to meet demand".format( - key[0], key[1], key[2]) - - logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") - return prob - - -# =============================================================================== - - -def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - - # create_constraint_max_flow_out_of_supply_vertex - # primary vertices only - # flow out of a vertex <= supply of the vertex, true for every day and commodity - - # for each primary (non-storage) supply vertex - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row_a in db_cur.execute("""select vertex_id, activity_level, supply - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and ft.facility_type = 'raw_material_producer' - and storage_vertex = 0;"""): - supply_vertex_id = row_a[0] - activity_level = row_a[1] - max_daily_supply = row_a[2] - actual_vertex_supply = activity_level * max_daily_supply - - flow_out = [] - db_cur2 = main_db_con.cursor() - # select all edges leaving that vertex and sum their flows - # should be a single connector edge - for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): - edge_id = row_b[0] - flow_out.append(flow_var[edge_id]) - - prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format( - actual_vertex_supply, supply_vertex_id) - # could easily add human-readable vertex info to this if desirable - - logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") - return prob - - -# =============================================================================== - - -def create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_var, processor_build_vars, - processor_daily_flow_vars): - logger.debug("STARTING: create_constraint_daily_processor_capacity") - import pdb - # pdb.set_trace() - # primary vertices only - # flow into vertex is capped at facility max_capacity per day - # sum over all input commodities, grouped by day and facility - # conservation of flow and ratios are handled in other methods - - ### get primary processor vertex and its input quantityi - total_scenario_min_capacity = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - sql = """select f.facility_id, - ifnull(f.candidate, 0), ifnull(f.max_capacity, -1), v.schedule_day, v.activity_level - from facility_commodities fc, facility_type_id ft, facilities f, vertices v - where ft.facility_type = 'processor' - and ft.facility_type_id = f.facility_type_id - and f.facility_id = fc.facility_id - and fc.io = 'i' - and v.facility_id = f.facility_id - and v.storage_vertex = 0 - group by f.facility_id, ifnull(f.candidate, 0), f.max_capacity, v.schedule_day, v.activity_level - ; - """ - # iterate through processor facilities, one constraint per facility per day - # no handling of subcommodities - - processor_facilities = db_cur.execute(sql) - - processor_facilities = processor_facilities.fetchall() - - for row_a in processor_facilities: - - # input_commodity_id = row_a[0] - facility_id = row_a[0] - is_candidate = row_a[1] - max_capacity = row_a[2] - day = row_a[3] - daily_activity_level = row_a[4] - - if max_capacity >= 0: - daily_inflow_max_capacity = float(max_capacity) * float(daily_activity_level) - daily_inflow_min_capacity = daily_inflow_max_capacity / 2 - logger.debug( - "processor {}, day {}, input capacity min: {} max: {}".format(facility_id, day, daily_inflow_min_capacity, - daily_inflow_max_capacity)) - total_scenario_min_capacity = total_scenario_min_capacity + daily_inflow_min_capacity - flow_in = [] - - # all edges that end in that processor facility primary vertex, on that day - db_cur2 = main_db_con.cursor() - for row_b in db_cur2.execute("""select edge_id from edges e, vertices v - where e.start_day = {} - and e.d_vertex_id = v.vertex_id - and v.facility_id = {} - and v.storage_vertex = 0 - group by edge_id""".format(day, facility_id)): - input_edge_id = row_b[0] - flow_in.append(flow_var[input_edge_id]) - - logger.debug( - "flow in for capacity constraint on processor facility {} day {}: {}".format(facility_id, day, flow_in)) - prob += lpSum(flow_in) <= daily_inflow_max_capacity * processor_daily_flow_vars[(facility_id, day)], \ - "constraint max flow into processor facility {}, day {}, flow var {}".format( - facility_id, day, processor_daily_flow_vars[facility_id, day]) - - prob += lpSum(flow_in) >= daily_inflow_min_capacity * processor_daily_flow_vars[ - (facility_id, day)], "constraint min flow into processor {}, day {}".format(facility_id, day) - # else: - # pdb.set_trace() - - if is_candidate == 1: - # forces processor build var to be correct - # if there is flow through a candidate processor then it has to be built - prob += processor_build_vars[facility_id] >= processor_daily_flow_vars[ - (facility_id, day)], "constraint forces processor build var to be correct {}, {}".format( - facility_id, processor_build_vars[facility_id]) - - logger.debug("FINISHED: create_constraint_daily_processor_capacity") - return prob - - -# =============================================================================== - - -def create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_primary_processor_vertex_constraints - conservation of flow") - # for all of these vertices, flow in always == flow out - # node_counter = 0 - # node_constraint_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - # total flow in == total flow out, subject to conversion; - # dividing by "required quantity" functionally converts all commodities to the same "processor-specific units" - - # processor primary vertices with input commodity and quantity needed to produce specified output quantities - # 2 sets of constraints; one for the primary processor vertex to cover total flow in and out - # one for each input and output commodity (sum over sources) to ensure its ratio matches facility_commodities - - # the current construction of this method is dependent on having only one input commodity type per processor - # this limitation makes sharing max transport distance from the input to an output commodity feasible - - logger.debug("conservation of flow and commodity ratios, primary processor vertices:") - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' - when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day - when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - ifnull(f.candidate, 0) candidate_check, - e.source_facility_id, - v.source_facility_id, - v.commodity_id, - c.share_max_transport_distance - from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f - join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) - where ft.facility_type = 'processor' - and v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 0 - and v.facility_id = fc.facility_id - and fc.commodity_id = c.commodity_id - and fc.commodity_id = e.commodity_id - group by v.vertex_id, - in_or_out_edge, - constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - candidate_check, - e.source_facility_id, - v.commodity_id, - v.source_facility_id, - ifnull(c.share_max_transport_distance, 'N') - order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id - ;""" - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - sql_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info( - "execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - sql_data = sql_data.fetchall() - logger.info( - "fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - # Nested dictionaries - # flow_in_lists[primary_processor_vertex_id] = dict of commodities handled by that processor vertex - - # flow_in_lists[primary_processor_vertex_id][commodity1] = - # list of edge ids that flow that commodity into that vertex - - # flow_in_lists[vertex_id].values() to get all flow_in edges for all commodities, a list of lists - # if edge out commodity inherits transport distance, then source_facility id must match. if not, aggregate - - flow_in_lists = {} - flow_out_lists = {} - inherit_max_transport = {} - # inherit_max_transport[commodity_id] = 'Y' or 'N' - - for row_a in sql_data: - - vertex_id = row_a[0] - in_or_out_edge = row_a[1] - # constraint_day = row_a[2] - commodity_id = row_a[3] - # mode = row_a[4] - edge_id = row_a[5] - # nx_edge_id = row_a[6] - quantity = float(row_a[7]) - # facility_id = row_a[8] - # commodity_name = row_a[9] - # fc_io_commodity = row_a[10] - # activity_level = row_a[11] - # is_candidate = row_a[12] - edge_source_facility_id = row_a[13] - vertex_source_facility_id = row_a[14] - # v_commodity_id = row_a[15] - inherit_max_transport_distance = row_a[16] - if commodity_id not in inherit_max_transport.keys(): - if inherit_max_transport_distance == 'Y': - inherit_max_transport[commodity_id] = 'Y' - else: - inherit_max_transport[commodity_id] = 'N' - - if in_or_out_edge == 'in': - # if the vertex isn't in the main dict yet, add it - # could have multiple source facilities - # could also have more than one input commodity now - flow_in_lists.setdefault(vertex_id, {}) - flow_in_lists[vertex_id].setdefault((commodity_id, quantity, edge_source_facility_id), []).append(flow_var[edge_id]) - # flow_in_lists[vertex_id] is itself a dict keyed on commodity, quantity (ratio) and edge_source_facility; - # value is a list of edge ids into that vertex of that commodity and edge source - - elif in_or_out_edge == 'out': - # for out-lists, could have multiple commodities as well as multiple sources - # some may have a max transport distance, inherited or independent, some may not - flow_out_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it - flow_out_lists[vertex_id].setdefault((commodity_id, quantity, edge_source_facility_id), []).append(flow_var[edge_id]) - - # Because we keyed on commodity, source facility tracking is merged as we pass through the processor vertex - - # 1) for each output commodity, check against an input to ensure correct ratio - only need one input - # 2) for each input commodity, check against an output to ensure correct ratio - only need one output; - # 2a) first sum sub-flows over input commodity - - # 1---------------------------------------------------------------------- - constrained_input_flow_vars = set([]) - # pdb.set_trace() - - for key, value in iteritems(flow_out_lists): - #value is a dictionary with commodity & source as keys - # set up a dictionary that will be filled with input lists to check ratio against - compare_input_dict = {} - compare_input_dict_commod = {} - vertex_id = key - zero_in = False - #value is a dictionary keyed on output commodity, quantity required, edge source - if vertex_id in flow_in_lists: - in_quantity = 0 - in_commodity_id = 0 - in_source_facility_id = -1 - for ikey, ivalue in iteritems(flow_in_lists[vertex_id]): - in_commodity_id = ikey[0] - in_quantity = ikey[1] - in_source = ikey[2] - # list of edges - compare_input_dict[in_source] = ivalue - # to accommodate and track multiple input commodities; does not keep sources separate - # aggregate lists over sources, by commodity - if in_commodity_id not in compare_input_dict_commod.keys(): - compare_input_dict_commod[in_commodity_id] = set([]) - for edge in ivalue: - compare_input_dict_commod[in_commodity_id].add(edge) - else: - zero_in = True - - - # value is a dict - we loop once here for each output commodity and source at the vertex - for key2, value2 in iteritems(value): - out_commodity_id = key2[0] - out_quantity = key2[1] - out_source = key2[2] - # edge_list = value2 - flow_var_list = value2 - # if we need to match source facility, there is only one set of input lists - # otherwise, use all input lists - this aggregates sources - # need to keep commodities separate, units may be different - # known issue - we could have double-counting problems if only some outputs have to inherit max - # transport distance through this facility - match_source = inherit_max_transport[out_commodity_id] - compare_input_list = [] - if match_source == 'Y': - if len(compare_input_dict_commod.keys()) > 1: - error = "Multiple input commodities for processors and shared max transport distance are" \ - " not supported within the same scenario." - logger.error(error) - raise Exception(error) - - if out_source in compare_input_dict.keys(): - compare_input_list = compare_input_dict[out_source] - # if no valid input edges - none for vertex, or if output needs to match source and there are no - # matching source - if zero_in or (match_source == 'Y' and len(compare_input_list) == 0): - prob += lpSum( - flow_var_list) == 0, "processor flow, vertex {} has zero in so zero out of commodity {} " \ - "with source {} if applicable".format( - vertex_id, out_commodity_id, out_source) - else: - if match_source == 'Y': - # ratio constraint for this output commodity relative to total input of each commodity - required_flow_out = lpSum(flow_var_list) / out_quantity - # check against an input dict - prob += required_flow_out == lpSum( - compare_input_list) / in_quantity, "processor flow, vertex {}, source_facility {}," \ - " commodity {} output quantity" \ - " checked against single input commodity quantity".format( - vertex_id, out_source, out_commodity_id, in_commodity_id) - for flow_var in compare_input_list: - constrained_input_flow_vars.add(flow_var) - else: - for k, v in iteritems(compare_input_dict_commod): - # pdb.set_trace() - # as long as the input source doesn't match an output that needs to inherit - compare_input_list = list(v) - in_commodity_id = k - # ratio constraint for this output commodity relative to total input of each commodity - required_flow_out = lpSum(flow_var_list) / out_quantity - # check against an input dict - prob += required_flow_out == lpSum( - compare_input_list) / in_quantity, "processor flow, vertex {}, source_facility {}," \ - " commodity {} output quantity" \ - " checked against commodity {} input quantity".format( - vertex_id, out_source, out_commodity_id, in_commodity_id) - for flow_var in compare_input_list: - constrained_input_flow_vars.add(flow_var) - - for key, value in iteritems(flow_in_lists): - vertex_id = key - for key2, value2 in iteritems(value): - commodity_id = key2[0] - # out_quantity = key2[1] - source = key2[2] - # edge_list = value2 - flow_var_list = value2 - for flow_var in flow_var_list: - if flow_var not in constrained_input_flow_vars: - prob += flow_var == 0, "processor flow, vertex {} has no matching out edges so zero in of " \ - "commodity {} with source {}".format( - vertex_id, commodity_id, source) - - logger.debug("FINISHED: create_primary_processor_conservation_of_flow_constraints") - return prob - - -# =============================================================================== - - -def create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_var, processor_excess_vars): - logger.debug("STARTING: create_constraint_conservation_of_flow") - # node_counter = 0 - node_constraint_counter = 0 - storage_vertex_constraint_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - logger.info("conservation of flow, storage vertices:") - # storage vertices, any facility type - # these have at most one direction of transport edges, so no need to track mode - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' - when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day - when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id, v.facility_id, c.commodity_name, - v.activity_level, - ft.facility_type - - from vertices v, facility_type_id ft, commodities c, facilities f - join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) - and (e.o_vertex_id = v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) - - where v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 1 - and v.commodity_id = c.commodity_id - - group by v.vertex_id, - in_or_out_edge, - constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id,v.facility_id, c.commodity_name, - v.activity_level - - order by v.facility_id, v.vertex_id, e.edge_id - ;""" - - # get the data from sql and see how long it takes. - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - vertexid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - vertexid_data = vertexid_data.fetchall() - logger.info( - "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_in_lists = {} - flow_out_lists = {} - for row_v in vertexid_data: - vertex_id = row_v[0] - in_or_out_edge = row_v[1] - constraint_day = row_v[2] - commodity_id = row_v[3] - edge_id = row_v[4] - # nx_edge_id = row_v[5] - # facility_id = row_v[6] - # commodity_name = row_v[7] - # activity_level = row_v[8] - facility_type = row_v[9] - - if in_or_out_edge == 'in': - flow_in_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( - flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( - flow_var[edge_id]) - - logger.info("adding processor excess variabless to conservation of flow") - - # add any processors to flow_out_lists if not already created - # Addresses bug documented in #382 - for key, value in iteritems(flow_in_lists): - facility_type = key[3] - if facility_type == 'processor' and key not in flow_out_lists.keys(): - flow_out_lists[key] = [] - - for key, value in iteritems(flow_out_lists): - vertex_id = key[0] - # commodity_id = key[1] - # day = key[2] - facility_type = key[3] - if facility_type == 'processor': - flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) - - for key, value in iteritems(flow_out_lists): - - if key in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum( - flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - else: - prob += lpSum(flow_out_lists[key]) == lpSum( - 0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - for key, value in iteritems(flow_in_lists): - - if key not in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum( - 0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - logger.info( - "total conservation of flow constraints created on nodes: {}".format(storage_vertex_constraint_counter)) - - logger.info("conservation of flow, nx_nodes:") - # for each day, get all edges in and out of the node. - # Sort edges by commodity and whether they're going in or out of the node - sql = """select nn.node_id, - (case when e.from_node_id = nn.node_id then 'out' - when e.to_node_id = nn.node_id then 'in' else 'error' end) in_or_out_edge, - (case when e.from_node_id = nn.node_id then start_day - when e.to_node_id = nn.node_id then end_day else 0 end) constraint_day, - e.commodity_id, - ifnull(mode, 'NULL'), - e.edge_id, nx_edge_id, - miles, - (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, - e.source_facility_id, - e.commodity_id - from networkx_nodes nn - join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) - where nn.location_id is null - order by nn.node_id, e.commodity_id, - (case when e.from_node_id = nn.node_id then start_day - when e.to_node_id = nn.node_id then end_day else 0 end), - in_or_out_edge, e.source_facility_id, e.commodity_id - ;""" - - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - nodeid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info( - "execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t " - "".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - nodeid_data = nodeid_data.fetchall() - logger.info( - "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_in_lists = {} - flow_out_lists = {} - - for row_a in nodeid_data: - node_id = row_a[0] - in_or_out_edge = row_a[1] - constraint_day = row_a[2] - # commodity_id = row_a[3] - mode = row_a[4] - edge_id = row_a[5] - # nx_edge_id = row_a[6] - # miles = row_a[7] - intermodal = row_a[8] - source_facility_id = row_a[9] - commodity_id = row_a[10] - - # node_counter = node_counter +1 - # if node is not intermodal, conservation of flow holds per mode; - # if intermodal, then across modes - if intermodal == 'N': - if in_or_out_edge == 'in': - flow_in_lists.setdefault( - (node_id, intermodal, source_facility_id, constraint_day, commodity_id, mode), []).append( - flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault( - (node_id, intermodal, source_facility_id, constraint_day, commodity_id, mode), []).append( - flow_var[edge_id]) - else: - if in_or_out_edge == 'in': - flow_in_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id), - []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id), - []).append(flow_var[edge_id]) - - for key, value in iteritems(flow_out_lists): - node_id = key[0] - # intermodal_flag = key[1] - source_facility_id = key[2] - day = key[3] - commodity_id = key[4] - if len(key) == 6: - node_mode = key[5] - else: - node_mode = 'intermodal' - if key in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[ - key]), "conservation of flow, nx node {}, " \ - "source facility {}, commodity {}, " \ - "day {}, mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - else: - prob += lpSum(flow_out_lists[key]) == lpSum( - 0), "conservation of flow (zero out), nx node {}, source facility {}, commodity {}, day {}," \ - " mode {}".format(node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - for key, value in iteritems(flow_in_lists): - node_id = key[0] - # intermodal_flag = key[1] - source_facility_id = key[2] - day = key[3] - commodity_id = key[4] - if len(key) == 6: - node_mode = key[5] - else: - node_mode = 'intermodal' - - if key not in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum( - 0), "conservation of flow (zero in), nx node {}, source facility {}, commodity {}, day {}," \ - " mode {}".format(node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - logger.info("total conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) - - # Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints - - logger.debug("FINISHED: create_constraint_conservation_of_flow") - - return prob - - -# =============================================================================== - - -def create_constraint_max_route_capacity(logger, the_scenario, prob, flow_var): - logger.info("STARTING: create_constraint_max_route_capacity") - logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) - # min_capacity_level must be a number from 0 to 1, inclusive - # min_capacity_level is only relevant when background flows are turned on - # it sets a floor to how much capacity can be reduced by volume. - # min_capacity_level = .25 means route capacity will never be less than 25% of full capacity, - # even if "volume" would otherwise restrict it further - # min_capacity_level = 0 allows a route to be made unavailable for FTOT flow if base volume is too high - # this currently applies to all modes - logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - # capacity for storage routes - sql = """select - rr.route_id, sr.storage_max, sr.route_name, e.edge_id, e.start_day - from route_reference rr - join storage_routes sr on sr.route_name = rr.route_name - join edges e on rr.route_id = e.route_id - ;""" - # get the data from sql and see how long it takes. - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - storage_edge_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for storage edges:") - logger.info("execute for edges for storage - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - storage_edge_data = storage_edge_data.fetchall() - logger.info("fetchall edges for storage - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in storage_edge_data: - route_id = row_a[0] - aggregate_storage_capac = row_a[1] - storage_route_name = row_a[2] - edge_id = row_a[3] - start_day = row_a[4] - - flow_lists.setdefault((route_id, aggregate_storage_capac, storage_route_name, start_day), []).append( - flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on storage route {} named {} for day {}".format(key[0], - key[2], - key[3]) - - logger.debug("route_capacity constraints created for all storage routes") - - # capacity for transport routes - # Assumption - all flowing material is in kgal, all flow is summed on a single non-pipeline nx edge - sql = """select e.edge_id, e.nx_edge_id, e.max_edge_capacity, e.start_day, e.simple_mode, e.phase_of_matter, - e.capac_minus_volume_zero_floor - from edges e - where e.max_edge_capacity is not null - and e.simple_mode != 'pipeline' - ;""" - # get the data from sql and see how long it takes. - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - route_capac_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for transport edges:") - logger.info("execute for non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - route_capac_data = route_capac_data.fetchall() - logger.info("fetchall non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in route_capac_data: - edge_id = row_a[0] - nx_edge_id = row_a[1] - nx_edge_capacity = row_a[2] - start_day = row_a[3] - simple_mode = row_a[4] - phase_of_matter = row_a[5] - capac_minus_background_flow = max(row_a[6], 0) - min_restricted_capacity = max(capac_minus_background_flow, nx_edge_capacity * the_scenario.minCapacityLevel) - - if simple_mode in the_scenario.backgroundFlowModes: - use_capacity = min_restricted_capacity - else: - use_capacity = nx_edge_capacity - - # flow is in thousand gallons (kgal), for liquid, or metric tons, for solid - # capacity is in truckload, rail car, barge, or pipeline movement per day - # if mode is road and phase is liquid, capacity is in truckloads per day, we want it in kgal - # ftot_supporting_gis tells us that there are 8 kgal per truckload, - # so capacity * 8 gives us correct units or kgal per day - # => use capacity * ftot_supporting_gis multiplier to get capacity in correct flow units - - multiplier = 1 # if units match, otherwise specified here - if simple_mode == 'road': - if phase_of_matter == 'liquid': - multiplier = the_scenario.truck_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.truck_load_solid.magnitude - elif simple_mode == 'water': - if phase_of_matter == 'liquid': - multiplier = the_scenario.barge_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.barge_load_solid.magnitude - elif simple_mode == 'rail': - if phase_of_matter == 'liquid': - multiplier = the_scenario.railcar_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.railcar_load_solid.magnitude - - converted_capacity = use_capacity * multiplier - - flow_lists.setdefault((nx_edge_id, converted_capacity, start_day), []).append(flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on nx edge {} for day {}".format(key[0], key[2]) - - logger.debug("route_capacity constraints created for all non-pipeline transport routes") - - logger.debug("FINISHED: create_constraint_max_route_capacity") - return prob - - -# =============================================================================== - - -def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_pipeline_capacity") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) - logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - # capacity for pipeline tariff routes - # with sasc, may have multiple flows per segment, slightly diff commodities - sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow allowed_flow, - l.source, e.mode, instr(e.mode, l.source) - from edges e, pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where e.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(e.mode, l.source)>0 - group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source - ;""" - # capacity needs to be shared over link_id for any edge_id associated with that link - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - pipeline_capac_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for transport edges:") - logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - pipeline_capac_data = pipeline_capac_data.fetchall() - logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in pipeline_capac_data: - edge_id = row_a[0] - # tariff_id = row_a[1] - link_id = row_a[2] - # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall - # Link capacity * 42 is now in kgal per day, to match flow in kgal - link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[3] - start_day = row_a[4] - capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[5], 0) - min_restricted_capacity = max(capac_minus_background_flow_kgal, - link_capacity_kgal_per_day * the_scenario.minCapacityLevel) - - # capacity_nodes_mode_source = row_a[6] - edge_mode = row_a[7] - # mode_match_check = row_a[8] - if 'pipeline' in the_scenario.backgroundFlowModes: - link_use_capacity = min_restricted_capacity - else: - link_use_capacity = link_capacity_kgal_per_day - - # add flow from all relevant edges, for one start; may be multiple tariffs - flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format( - key[0], key[3], key[2]) - - logger.debug("pipeline capacity constraints created for all transport routes") - - logger.debug("FINISHED: create_constraint_pipeline_capacity") - return prob - - -# =============================================================================== - - -def setup_pulp_problem(the_scenario, logger): - logger.info("START: setup PuLP problem") - - # flow_var is the flow on each edge by commodity and day. - # the optimal value of flow_var will be solved by PuLP - flow_vars = create_flow_vars(the_scenario, logger) - - # unmet_demand_var is the unmet demand at each destination, being determined - unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) - - # processor_build_vars is the binary variable indicating whether a candidate processor is used - # and thus whether its build cost is charged - processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) - - # binary tracker variables - processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario, logger) - - # tracking unused production - processor_excess_vars = create_processor_excess_output_vars(the_scenario, logger) - - # THIS IS THE OBJECTIVE FUNCTION FOR THE OPTIMIZATION - # ================================================== - - prob = create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) - - prob = create_constraint_unmet_demand(logger, the_scenario, prob, flow_vars, unmet_demand_vars) - - prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) - - # This constraint is being excluded because 1) it is not used in current scenarios and 2) it is not supported by - # this version - it conflicts with the change permitting multiple inputs - # adding back 12/2020 - prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, - processor_vertex_flow_vars) - - prob = create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_vars) - - prob = create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_vars, processor_excess_vars) - - if the_scenario.capacityOn: - prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) - - prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) - - del unmet_demand_vars - - del flow_vars - - # The problem data is written to an .lp file - prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_c2.lp")) - - logger.info("FINISHED: setup PuLP problem") - return prob - - -# =============================================================================== - - -def solve_pulp_problem(prob_final, the_scenario, logger): - import datetime - - logger.info("START: solve_pulp_problem") - start_time = datetime.datetime.now() - from os import dup, dup2, close - f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') - orig_std_out = dup(1) - dup2(f.fileno(), 1) - - # status = prob_final.solve (PULP_CBC_CMD(maxSeconds = i_max_sec, fracGap = d_opt_gap, msg=1)) - # CBC time limit and relative optimality gap tolerance - status = prob_final.solve(PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance - logger.info('Completion code: %d; Solution status: %s; Best obj value found: %s' % ( - status, LpStatus[prob_final.status], value(prob_final.objective))) - - dup2(orig_std_out, 1) - close(orig_std_out) - f.close() - # The problem is solved using PuLP's choice of Solver - - logger.info("completed calling prob.solve()") - logger.info( - "FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - # THIS IS THE SOLUTION - - # The status of the solution is printed to the screen - ##LpStatus key string value numerical value - ##LpStatusOptimal ?Optimal? 1 - ##LpStatusNotSolved ?Not Solved? 0 - ##LpStatusInfeasible ?Infeasible? -1 - ##LpStatusUnbounded ?Unbounded? -2 - ##LpStatusUndefined ?Undefined? -3 - logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) - - logger.result( - "Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format( - float(value(prob_final.objective)))) - - return prob_final - - -# =============================================================================== - -def save_pulp_solution(the_scenario, prob, logger, zero_threshold=0.00001): - import datetime - start_time = datetime.datetime.now() - logger.info("START: save_pulp_solution") - non_zero_variable_count = 0 - - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_cur = db_con.cursor() - # drop the optimal_solution table - # ----------------------------- - db_cur.executescript("drop table if exists optimal_solution;") - - # create the optimal_solution table - # ----------------------------- - db_cur.executescript(""" - create table optimal_solution - ( - variable_name string, - variable_value real - ); - """) - - # insert the optimal data into the DB - # ------------------------------------- - for v in prob.variables(): - if v.varValue is None: - logger.debug("Variable value is none: " + str(v.name)) - else: - if v.varValue > zero_threshold: # eliminates values too close to zero - sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format( - v.name, float(v.varValue)) - db_con.execute(sql) - non_zero_variable_count = non_zero_variable_count + 1 - - # query the optimal_solution table in the DB for each variable we care about - # ---------------------------------------------------------------------------- - sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_processors_count = data.fetchone()[0] - logger.info("number of optimal_processors: {}".format(optimal_processors_count)) - - sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_count = data.fetchone()[0] - logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) - sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_sum = data.fetchone()[0] - logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) - logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) - logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format( - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) - - - sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" - data = db_con.execute(sql) - optimal_edges_count = data.fetchone()[0] - logger.info("number of optimal edges: {}".format(optimal_edges_count)) - - logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format( - float(value(prob.objective)) - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) - logger.info( - "Total Scenario Cost = (transportation + unmet demand penalty + " - "processor construction): \t ${0:,.0f}".format( - float(value(prob.objective)))) - - logger.info( - "FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - -# =============================================================================== - -def record_pulp_solution(the_scenario, logger): - logger.info("START: record_pulp_solution") - non_zero_variable_count = 0 - - with sqlite3.connect(the_scenario.main_db) as db_con: - - logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) - sql = """ - create table optimal_variables as - select - 'UnmetDemand' as variable_type, - cast(substr(variable_name, 13) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - null as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as original_facility, - null as final_facility, - null as prior_edge, - null as miles_travelled - from optimal_solution - where variable_name like 'UnmetDemand%' - union - select - 'Edge' as variable_type, - cast(substr(variable_name, 6) as int) var_id, - variable_value, - edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, - edges.volume*edges.units_conversion_multiplier as converted_volume, - edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, - edges.edge_type, - commodities.commodity_name, - ov.facility_name as o_facility, - dv.facility_name as d_facility, - o_vertex_id, - d_vertex_id, - from_node_id, - to_node_id, - start_day time_period, - edges.commodity_id, - edges.source_facility_id, - s.source_facility_name, - commodities.units, - variable_name, - edges.nx_edge_id, - edges.mode, - edges.mode_oid, - edges.miles, - null as original_facility, - null as final_facility, - null as prior_edge, - edges.miles_travelled as miles_travelled - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join commodities on edges.commodity_id = commodities.commodity_ID - left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id - left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id - left outer join source_commodity_ref as s on edges.source_facility_id = s.source_facility_id - where variable_name like 'Edge%' - union - select - 'BuildProcessor' as variable_type, - cast(substr(variable_name, 16) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - 'placeholder' as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as original_facility, - null as final_facility, - null as prior_edge, - null as miles_travelled - from optimal_solution - where variable_name like 'Build%'; - """ - db_con.execute("drop table if exists optimal_variables;") - db_con.execute(sql) - - logger.info("FINISH: record_pulp_solution") - -# =============================================================================== - - -def parse_optimal_solution_db(the_scenario, logger): - logger.info("starting parse_optimal_solution") - - optimal_processors = [] - optimal_processor_flows = [] - optimal_route_flows = {} - optimal_unmet_demand = {} - optimal_storage_flows = {} - optimal_excess_material = {} - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # do the Storage Edges - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" - data = db_con.execute(sql) - optimal_storage_edges = data.fetchall() - for edge in optimal_storage_edges: - optimal_storage_flows[edge] = optimal_storage_edges[edge] - - # do the Route Edges - sql = """select - variable_name, variable_value, - cast(substr(variable_name, 6) as int) edge_id, - route_id, start_day time_period, edges.commodity_id, - o_vertex_id, d_vertex_id, - v1.facility_id o_facility_id, - v2.facility_id d_facility_id - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join vertices v1 on edges.o_vertex_id = v1.vertex_id - join vertices v2 on edges.d_vertex_id = v2.vertex_id - where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; - """ - data = db_con.execute(sql) - optimal_route_edges = data.fetchall() - for edge in optimal_route_edges: - - variable_name = edge[0] - - variable_value = edge[1] - - edge_id = edge[2] - - route_id = edge[3] - - time_period = edge[4] - - commodity_flowed = edge[5] - - od_pair_name = "{}, {}".format(edge[8], edge[9]) - - # first time route_id is used on a day or commodity - if route_id not in optimal_route_flows: - optimal_route_flows[route_id] = [[od_pair_name, time_period, commodity_flowed, variable_value]] - - else: # subsequent times route is used on different day or for other commodity - optimal_route_flows[route_id].append([od_pair_name, time_period, commodity_flowed, variable_value]) - - # do the processors - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_candidates_processors = data.fetchall() - for proc in optimal_candidates_processors: - optimal_processors.append(proc) - - # do the processor vertex flows - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'ProcessorVertexFlow%';" - data = db_con.execute(sql) - optimal_processor_flows_sql = data.fetchall() - for proc in optimal_processor_flows_sql: - optimal_processor_flows.append(proc) - - # do the UnmetDemand - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmetdemand = data.fetchall() - for ultimate_destination in optimal_unmetdemand: - v_name = ultimate_destination[0] - v_value = ultimate_destination[1] - - search = re.search('\(.*\)', v_name.replace("'", "")) - - if search: - parts = search.group(0).replace("(", "").replace(")", "").split(",_") - - dest_name = parts[0] - commodity_flowed = parts[2] - if not dest_name in optimal_unmet_demand: - optimal_unmet_demand[dest_name] = {} - - if not commodity_flowed in optimal_unmet_demand[dest_name]: - optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) - else: - optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) - - - logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors - logger.info("length of optimal_processor_flows list: {}".format( - len(optimal_processor_flows))) # a list of optimal processor flows - logger.info("length of optimal_route_flows dict: {}".format( - len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values - logger.info("length of optimal_unmet_demand dict: {}".format( - len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values - - return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material - - +# --------------------------------------------------------------------------------------------------- +# Name: ftot_pulp +# +# Purpose: PulP optimization - create and run a modified facility location problem. +# Take NetworkX and GIS scenario data as recorded in main.db and convert to a structure of edges, nodes, vertices. +# Create variables for flow over edges, unmet demand, processor use, and candidate processors to build if present +# Solve cost minimization for unmet demand, transportation, and facility build costs +# Constraints ensure compliance with scenario requirements (e.g. max_route_capacity) +# as well as general problem structure (e.g. conservation_of_flow) +# --------------------------------------------------------------------------------------------------- + +import datetime +import pdb +import re +import sqlite3 +from collections import defaultdict +import os +from six import iteritems + +from pulp import * + +import ftot_supporting +from ftot_supporting import get_total_runtime_string +from ftot import Q_ + +# =================== constants============= +storage = 1 +primary = 0 +fixed_schedule_id = 2 +fixed_route_duration = 0 + +THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 + +storage_cost_1 = 0.01 +storage_cost_2 = 0.05 +facility_onsite_storage_max = 10000000000 +facility_onsite_storage_min = 0 + +default_max_capacity = 10000000000 +default_min_capacity = 0 + + +def o1(the_scenario, logger): + # create vertices, then edges for permitted modes, then set volume & capacity on edges + pre_setup_pulp(logger, the_scenario) + + +def o2(the_scenario, logger): + # create variables, problem to optimize, and constraints + prob = setup_pulp_problem(the_scenario, logger) + prob = solve_pulp_problem(prob, the_scenario, logger) + save_pulp_solution(the_scenario, prob, logger) + record_pulp_solution(the_scenario, logger) + from ftot_supporting import post_optimization + post_optimization(the_scenario, 'o2', logger) + + +# =============================================================================== + + +# helper function that reads in schedule data and returns dict of Schedule objects +def generate_schedules(the_scenario, logger): + logger.debug("start: generate_schedules") + default_availabilities = {} + day_availabilities = {} + last_day = 1 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + schedule_data = db_cur.execute(""" + select schedule_id, + day, + availability + + from schedules""") + + schedule_data = schedule_data.fetchall() + for row_a in schedule_data: + schedule_id = row_a[0] + day = row_a[1] + availability = float(row_a[2]) + + if day == 0: # denotes the default value + default_availabilities[schedule_id] = availability + elif schedule_id in day_availabilities.keys(): + day_availabilities[schedule_id][day] = availability + else: + day_availabilities[schedule_id] = {day: availability} # initialize sub-dic + # last_day is the greatest day specified across ALL schedules to avoid mismatch of length + if day > last_day: + last_day = day + + # make dictionary to store schedule objects + schedule_dict = {} + + # after reading in csv, parse data into dictionary object + for schedule_id in default_availabilities.keys(): + # initialize list of length + schedule_dict[schedule_id] = [default_availabilities[schedule_id] for i in range(last_day)] + # change different days to actual availability instead of the default values + # schedule_id may not be in day_availabilities if schedule has default value on all days + if schedule_id in day_availabilities.keys(): + for day in day_availabilities[schedule_id].keys(): + # schedule_dict[schedule + schedule_dict[schedule_id][day-1] = day_availabilities[schedule_id][day] + + for key in schedule_dict.keys(): + logger.debug("schedule name: " + str(key)) + logger.debug("availability: ") + logger.debug(schedule_dict[key]) + logger.debug("finished: generate_schedules") + + return schedule_dict, last_day + + +def make_vehicle_type_dict(the_scenario, logger): + + # check for vehicle type file + ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) + vehicle_types_path = os.path.join(ftot_program_directory, "lib", "vehicle_types.csv") + if not os.path.exists(vehicle_types_path): + logger.warning("warning: cannot find vehicle_types file: {}".format(vehicle_types_path)) + return {} # return empty dict + + # initialize vehicle property dict and read through vehicle_types CSV + vehicle_dict = {} + with open(vehicle_types_path, 'r') as vt: + line_num = 1 + for line in vt: + if line_num == 1: + pass # do nothing + else: + flds = line.rstrip('\n').split(',') + vehicle_label = flds[0] + mode = flds[1].lower() + vehicle_property = flds[2] + property_value = flds[3] + + # validate entries + assert vehicle_label not in ['Default', 'NA'], "Vehicle label: {} is a protected word. Please rename the vehicle.".format(vehicle_label) + + assert mode in ['road', 'water', 'rail'], "Mode: {} is not supported. Please specify road, water, or rail.".format(mode) + + assert vehicle_property in ['Truck_Load_Solid', 'Railcar_Load_Solid', 'Barge_Load_Solid', 'Truck_Load_Liquid', + 'Railcar_Load_Liquid', 'Barge_Load_Liquid', 'Pipeline_Crude_Load_Liquid', 'Pipeline_Prod_Load_Liquid', + 'Truck_Fuel_Efficiency', 'Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', + 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted', 'Barge_Fuel_Efficiency', + 'Barge_CO2_Emissions', 'Rail_Fuel_Efficiency', 'Railroad_CO2_Emissions'], \ + "Vehicle property: {} is not recognized. Refer to scenario.xml for supported property labels.".format(vehicle_property) + + # convert units + # Pint throws an exception if units are invalid + if vehicle_property in ['Truck_Load_Solid', 'Railcar_Load_Solid', 'Barge_Load_Solid']: + # convert csv value into default solid units + property_value = Q_(property_value).to(the_scenario.default_units_solid_phase) + elif vehicle_property in ['Truck_Load_Liquid', 'Railcar_Load_Liquid', 'Barge_Load_Liquid', 'Pipeline_Crude_Load_Liquid', 'Pipeline_Prod_Load_Liquid']: + # convert csv value into default liquid units + property_value = Q_(property_value).to(the_scenario.default_units_liquid_phase) + elif vehicle_property in ['Truck_Fuel_Efficiency', 'Barge_Fuel_Efficiency', 'Rail_Fuel_Efficiency']: + # convert csv value into miles per gallon + property_value = Q_(property_value).to('mi/gal') + elif vehicle_property in ['Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted']: + # convert csv value into grams per mile + property_value = Q_(property_value).to('g/mi') + elif vehicle_property in ['Barge_CO2_Emissions', 'Railroad_CO2_Emissions']: + # convert csv value into grams per default mass unit per mile + property_value = Q_(property_value).to('g/{}/mi'.format(the_scenario.default_units_solid_phase)) + else: + pass # do nothing + + # populate dictionary + if mode not in vehicle_dict: + # create entry for new mode type + vehicle_dict[mode] = {} + if vehicle_label not in vehicle_dict[mode]: + # create new vehicle key and add property + vehicle_dict[mode][vehicle_label] = {vehicle_property: property_value} + else: + # add property to existing vehicle + if vehicle_property in vehicle_dict[mode][vehicle_label].keys(): + logger.warning('Property: {} already exists for Vehicle: {}. Overwriting with value: {}'.\ + format(vehicle_property, vehicle_label, property_value)) + vehicle_dict[mode][vehicle_label][vehicle_property] = property_value + + line_num += 1 + + # ensure all properties are included + for mode in vehicle_dict: + if mode == 'road': + properties = ['Truck_Load_Solid', 'Truck_Load_Liquid', 'Truck_Fuel_Efficiency', + 'Atmos_CO2_Urban_Unrestricted', 'Atmos_CO2_Urban_Restricted', + 'Atmos_CO2_Rural_Unrestricted', 'Atmos_CO2_Rural_Restricted'] + elif mode == 'water': + properties = ['Barge_Load_Solid', 'Barge_Load_Liquid', 'Barge_Fuel_Efficiency', + 'Barge_CO2_Emissions'] + elif mode == 'rail': + properties = ['Railcar_Load_Solid', 'Railcar_Load_Liquid', 'Rail_Fuel_Efficiency', + 'Railroad_CO2_Emissions'] + for vehicle_label in vehicle_dict[mode]: + for required_property in properties: + assert required_property in vehicle_dict[mode][vehicle_label].keys(), "Property: {} missing from Vehicle: {}".format(required_property, vehicle_label) + + return vehicle_dict + + +def vehicle_type_setup(the_scenario, logger): + + logger.info("START: vehicle_type_setup") + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + main_db_con.executescript(""" + drop table if exists vehicle_types; + create table vehicle_types( + mode text, + vehicle_label text, + property_name text, + property_value text, + CONSTRAINT unique_vehicle_and_property UNIQUE(mode, vehicle_label, property_name)) + ;""") + + vehicle_dict = make_vehicle_type_dict(the_scenario, logger) + for mode in vehicle_dict: + for vehicle_label in vehicle_dict[mode]: + for vehicle_property in vehicle_dict[mode][vehicle_label]: + + property_value = vehicle_dict[mode][vehicle_label][vehicle_property] + + main_db_con.execute(""" + insert or ignore into vehicle_types + (mode, vehicle_label, property_name, property_value) + VALUES + ('{}','{}','{}','{}') + ; + """.format(mode, vehicle_label, vehicle_property, property_value)) + + +def make_commodity_mode_dict(the_scenario, logger): + + logger.info("START: make_commodity_mode_dict") + + if the_scenario.commodity_mode_data == "None": + logger.info('commodity_mode_data file not specified.') + return {} # return empty dict + + # check if path to table exists + elif not os.path.exists(the_scenario.commodity_mode_data): + logger.warning("warning: cannot find commodity_mode_data file: {}".format(the_scenario.commodity_mode_data)) + return {} # return empty dict + + # initialize dict and read through commodity_mode CSV + commodity_mode_dict = {} + with open(the_scenario.commodity_mode_data, 'r') as rf: + line_num = 1 + header = None # will assign within for loop + for line in rf: + if line_num == 1: + header = line.rstrip('\n').split(',') + # Replace the short pipeline name with the long name + for h in range(len(header)): + if header[h] == 'pipeline_crude': + header[h] = 'pipeline_crude_trf_rts' + elif header[h] == 'pipeline_prod': + header[h] = 'pipeline_prod_trf_rts' + else: + flds = line.rstrip('\n').split(',') + commodity_name = flds[0].lower() + assignment = flds[1:] + if commodity_name in commodity_mode_dict.keys(): + logger.warning('Commodity: {} already exists. Overwriting with assignments: {}'.format(commodity_name, assignment)) + commodity_mode_dict[commodity_name] = dict(zip(header[1:], assignment)) + line_num += 1 + + # warn if trying to permit a mode that is not permitted in the scenario + for commodity in commodity_mode_dict: + for mode in commodity_mode_dict[commodity]: + if commodity_mode_dict[commodity][mode] != 'N' and mode not in the_scenario.permittedModes: + logger.warning("Mode: {} not permitted in scenario. Commodity: {} will not travel on this mode".format(mode, commodity)) + + return commodity_mode_dict + + +def commodity_mode_setup(the_scenario, logger): + + logger.info("START: commodity_mode_setup") + + # set up vehicle types table + vehicle_type_setup(the_scenario, logger) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + main_db_con.executescript(""" + drop table if exists commodity_mode; + + create table commodity_mode( + mode text, + commodity_id text, + commodity_phase text, + vehicle_label text, + allowed_yn text, + CONSTRAINT unique_commodity_and_mode UNIQUE(commodity_id, mode)) + ;""") + + # query commmodities table + commod = main_db_con.execute("select commodity_name, commodity_id, phase_of_matter from commodities where commodity_name <> 'multicommodity';") + commod = commod.fetchall() + commodities = {} + for row in commod: + commodity_name = row[0] + commodity_id = row[1] + phase_of_matter = row[2] + commodities[commodity_name] = (commodity_id, phase_of_matter) + + # query vehicle types table + vehs = main_db_con.execute("select distinct mode, vehicle_label from vehicle_types;") + vehs = vehs.fetchall() + vehicle_types = {} + for row in vehs: + mode = row[0] + vehicle_label = row[1] + if mode not in vehicle_types: + # add new mode to dictionary and start vehicle list + vehicle_types[mode] = [vehicle_label] + else: + # append new vehicle to mode's vehicle list + vehicle_types[mode].append(vehicle_label) + + # assign mode permissions and vehicle labels to commodities + commodity_mode_dict = make_commodity_mode_dict(the_scenario, logger) + logger.info("----- commodity/vehicle type table -----") + + for permitted_mode in the_scenario.permittedModes: + for k, v in iteritems(commodities): + commodity_name = k + commodity_id = v[0] + phase_of_matter = v[1] + + allowed = 'Y' # may be updated later in loop + vehicle_label = 'Default' # may be updated later in loop + + if commodity_name in commodity_mode_dict and permitted_mode in commodity_mode_dict[commodity_name]: + + # get user's entry for commodity and mode + assignment = commodity_mode_dict[commodity_name][permitted_mode] + + if assignment == 'Y': + if phase_of_matter != 'liquid' and 'pipeline' in permitted_mode: + # solids not permitted on pipeline. + # note that FTOT previously asserts no custom vehicle label is created for pipeline + logger.warning("commodity {} is not liquid and cannot travel through pipeline mode: {}".format( + commodity_name, permitted_mode)) + allowed = 'N' + vehicle_label = 'NA' + + elif assignment == 'N': + allowed = 'N' + vehicle_label = 'NA' + + else: + # user specified a vehicle type + allowed = 'Y' + if permitted_mode in vehicle_types and assignment in vehicle_types[permitted_mode]: + # accept user's assignment + vehicle_label = assignment + else: + # assignment not a known vehicle. fail. + raise Exception("improper vehicle label in Commodity_Mode_Data_csv for commodity: {}, mode: {}, and vehicle: {}". \ + format(commodity_name, permitted_mode, assignment)) + + elif 'pipeline' in permitted_mode: + # for unspecified commodities, default to not permitted on pipeline + allowed = 'N' + vehicle_label = 'NA' + + logger.info("Commodity name: {}, Mode: {}, Allowed: {}, Vehicle type: {}". \ + format(commodity_name, permitted_mode, allowed, vehicle_label)) + + # write table. only includes modes that are permitted in the scenario xml file. + main_db_con.execute(""" + insert or ignore into commodity_mode + (mode, commodity_id, commodity_phase, vehicle_label, allowed_yn) + VALUES + ('{}',{},'{}','{}','{}') + ; + """.format(permitted_mode, commodity_id, phase_of_matter, vehicle_label, allowed)) + + return + + +# =============================================================================== + + +def source_tracking_setup(the_scenario, logger): + logger.info("START: source_tracking_setup") + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + max_inputs = main_db_con.execute("""select max(inputs) from + (select count(fc.commodity_id) inputs + from facility_commodities fc, facility_type_id ft + where ft.facility_type = 'processor' + and fc.io = 'i' + group by fc.facility_id) + ;""") + + for row in max_inputs: + if row[0] == None: + max_inputs_to_a_processor = 0 + else: + max_inputs_to_a_processor = int(row[0]) + if max_inputs_to_a_processor > 1: + logger.warning("error, this version of the optimization step only functions correctly with a single input" + " commodity type per processor") + + main_db_con.executescript(""" + + insert or ignore into commodities(commodity_name) values ('multicommodity'); + + drop table if exists source_commodity_ref + ; + + create table source_commodity_ref(id INTEGER PRIMARY KEY, + source_facility_id integer, + source_facility_name text, + source_facility_type_id integer, --lets us differentiate spiderweb from processor + commodity_id integer, + commodity_name text, + units text, + phase_of_matter text, + max_transport_distance numeric, + max_transport_distance_flag text, + share_max_transport_distance text, + CONSTRAINT unique_source_and_name UNIQUE(commodity_id, source_facility_id)) + ; + + insert or ignore into source_commodity_ref ( + source_facility_id, + source_facility_name, + source_facility_type_id, + commodity_id, + commodity_name, + units, + phase_of_matter, + max_transport_distance, + max_transport_distance_flag, + share_max_transport_distance) + select + f.facility_id, + f.facility_name, + f.facility_type_id, + c.commodity_id, + c.commodity_name, + c.units, + c.phase_of_matter, + (case when c.max_transport_distance is not null then + c.max_transport_distance else Null end) max_transport_distance, + (case when c.max_transport_distance is not null then 'Y' else 'N' end) max_transport_distance_flag, + (case when ifnull(c.share_max_transport_distance, 'N') = 'Y' then 'Y' else 'N' end) share_max_transport_distance + + from commodities c, facilities f, facility_commodities fc + where f.facility_id = fc.facility_id + and f.ignore_facility = 'false' + and fc.commodity_id = c.commodity_id + and fc.io = 'o' + and ifnull(c.share_max_transport_distance, 'N') != 'Y' + ; + + insert or ignore into source_commodity_ref ( + source_facility_id, + source_facility_name, + source_facility_type_id, + commodity_id, + commodity_name, + units, + phase_of_matter, + max_transport_distance, + max_transport_distance_flag, + share_max_transport_distance) + select + sc.source_facility_id, + sc.source_facility_name, + sc.source_facility_type_id, + o.commodity_id, + c.commodity_name, + c.units, + c.phase_of_matter, + sc.max_transport_distance, + sc.max_transport_distance_flag, + o.share_max_transport_distance + from source_commodity_ref sc, facility_commodities i, facility_commodities o, commodities c + where o.share_max_transport_distance = 'Y' + and sc.commodity_id = i.commodity_id + and o.facility_id = i.facility_id + and o.io = 'o' + and i.io = 'i' + and o.commodity_id = c.commodity_id + ; + """ + ) + + return + + +# =============================================================================== + + +def generate_all_vertices(the_scenario, schedule_dict, schedule_length, logger): + logger.info("START: generate_all_vertices table") + + total_potential_production = {} + multi_commodity_name = "multicommodity" + + storage_availability = 1 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + logger.debug("create the vertices table") + # create the vertices table + main_db_con.executescript(""" + drop table if exists vertices + ; + + create table if not exists vertices ( + vertex_id INTEGER PRIMARY KEY, location_id, + facility_id integer, facility_name text, facility_type_id integer, schedule_day integer, + commodity_id integer, activity_level numeric, storage_vertex binary, + udp numeric, supply numeric, demand numeric, + source_facility_id integer, + iob text, --allows input, output, or both + CONSTRAINT unique_vertex UNIQUE(facility_id, schedule_day, commodity_id, source_facility_id, storage_vertex)) + ;""") + + # create indexes for the networkx nodes and links tables + logger.info("create an index for the networkx nodes and links tables") + main_db_con.executescript(""" + CREATE INDEX IF NOT EXISTS node_index ON networkx_nodes (node_id, location_id) + ; + + create index if not exists nx_edge_index on + networkx_edges(from_node_id, to_node_id, + artificial, mode_source, mode_source_OID, + miles, route_cost_scaling, capacity) + ; + """) + + # -------------------------------- + + db_cur = main_db_con.cursor() + # nested cursor + db_cur4 = main_db_con.cursor() + counter = 0 + total_facilities = 0 + + for row in db_cur.execute("select count(distinct facility_id) from facilities;"): + total_facilities = row[0] + + # create vertices for each non-ignored facility facility + # facility_type can be "raw_material_producer", "ultimate_destination","processor"; + # get id from facility_type_id table + # any other facility types are not currently handled + + facility_data = db_cur.execute(""" + select facility_id, + facility_type, + facility_name, + location_id, + f.facility_type_id, + schedule_id + + from facilities f, facility_type_id ft + where ignore_facility = '{}' + and f.facility_type_id = ft.facility_type_id; + """.format('false')) + facility_data = facility_data.fetchall() + for row_a in facility_data: + + db_cur2 = main_db_con.cursor() + facility_id = row_a[0] + facility_type = row_a[1] + facility_name = row_a[2] + facility_location_id = row_a[3] + facility_type_id = row_a[4] + schedule_id = row_a[5] + if counter % 10000 == 1: + logger.info("vertices created for {} facilities of {}".format(counter, total_facilities)) + for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): + logger.info('{} vertices created'.format(row_d[0])) + counter = counter + 1 + + if facility_type == "processor": + # actual processors - will deal with endcaps in edges section + + # create processor vertices for any commodities that do not inherit max transport distance + proc_data = db_cur2.execute("""select fc.commodity_id, + ifnull(fc.quantity, 0), + fc.units, + ifnull(c.supertype, c.commodity_name), + fc.io, + mc.commodity_id, + c.commodity_name, + ifnull(s.source_facility_id, 0) + from facility_commodities fc, commodities c, commodities mc + left outer join source_commodity_ref s + on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') + where fc.facility_id = {} + and fc.commodity_id = c.commodity_id + and mc.commodity_name = '{}';""".format(facility_id, multi_commodity_name)) + + proc_data = proc_data.fetchall() + # entry for each incoming commodity and its potential sources + # each outgoing commodity with this processor as their source IF there is a max commod distance + for row_b in proc_data: + + commodity_id = row_b[0] + quantity = row_b[1] + io = row_b[4] + id_for_mult_commodities = row_b[5] + commodity_name = row_b[6] + source_facility_id = row_b[7] + new_source_facility_id = facility_id + + # vertices for generic demand type, or any subtype specified by the destination + for day_before, availability in enumerate(schedule_dict[schedule_id]): + if io == 'i': + main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, + facility_type_id, facility_name, schedule_day, commodity_id, activity_level, + storage_vertex, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + '{}' );""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + id_for_mult_commodities, availability, primary, + new_source_facility_id, 'b')) + + main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, + facility_type_id, facility_name, schedule_day, commodity_id, activity_level, + storage_vertex, demand, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, {}, + {}, {}, {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, + facility_name, + day_before+1, commodity_id, storage_availability, storage, quantity, + source_facility_id, io)) + + else: + if commodity_name != 'total_fuel': + main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, + facility_type_id, facility_name, schedule_day, commodity_id, activity_level, + storage_vertex, supply, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, + {}, {}, {}, {}, '{}');""".format( + facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, storage_availability, storage, quantity, source_facility_id, io)) + + + elif facility_type == "raw_material_producer": + rmp_data = db_cur.execute("""select fc.commodity_id, fc.quantity, fc.units, + ifnull(s.source_facility_id, 0), io + from facility_commodities fc + left outer join source_commodity_ref s + on (fc.commodity_id = s.commodity_id + and s.max_transport_distance_flag = 'Y' + and s.source_facility_id = {}) + where fc.facility_id = {};""".format(facility_id, facility_id)) + + rmp_data = rmp_data.fetchall() + + for row_b in rmp_data: + commodity_id = row_b[0] + quantity = row_b[1] + # units = row_b[2] + source_facility_id = row_b[3] + iob = row_b[4] + + if commodity_id in total_potential_production: + total_potential_production[commodity_id] = total_potential_production[commodity_id] + quantity + else: + total_potential_production[commodity_id] = quantity + + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + activity_level, storage_vertex, supply, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, availability, primary, quantity, + source_facility_id, iob)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + activity_level, storage_vertex, supply, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, storage_availability, storage, quantity, + source_facility_id, iob)) + + elif facility_type == "storage": # storage facility + storage_fac_data = db_cur.execute("""select + fc.commodity_id, + fc.quantity, + fc.units, + ifnull(s.source_facility_id, 0), + io + from facility_commodities fc + left outer join source_commodity_ref s + on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') + where fc.facility_id = {} ;""".format(facility_id)) + + storage_fac_data = storage_fac_data.fetchall() + + for row_b in storage_fac_data: + commodity_id = row_b[0] + source_facility_id = row_b[3] # 0 if not source-tracked + iob = row_b[4] + + for day_before in range(schedule_length): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + storage_vertex, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, storage, + source_facility_id, iob)) + + elif facility_type == "ultimate_destination": + + dest_data = db_cur2.execute("""select + fc.commodity_id, + ifnull(fc.quantity, 0), + fc.units, + fc.commodity_id, + ifnull(c.supertype, c.commodity_name), + ifnull(s.source_facility_id, 0), + io + from facility_commodities fc, commodities c + left outer join source_commodity_ref s + on (fc.commodity_id = s.commodity_id and s.max_transport_distance_flag = 'Y') + where fc.facility_id = {} + and fc.commodity_id = c.commodity_id;""".format(facility_id)) + + dest_data = dest_data.fetchall() + + for row_b in dest_data: + commodity_id = row_b[0] + quantity = row_b[1] + commodity_supertype = row_b[4] + source_facility_id = row_b[5] + iob = row_b[6] + zero_source_facility_id = 0 # material merges at primary vertex + + # vertices for generic demand type, or any subtype specified by the destination + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + activity_level, storage_vertex, demand, udp, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, + {}, {}, {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, availability, primary, quantity, + the_scenario.unMetDemandPenalty, + zero_source_facility_id, iob)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + activity_level, storage_vertex, demand, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, storage_availability, storage, quantity, + source_facility_id, iob)) + # vertices for other fuel subtypes that match the destination's supertype + # if the subtype is in the commodity table, it is produced by some facility in the scenario + db_cur3 = main_db_con.cursor() + for row_c in db_cur3.execute("""select commodity_id, units from commodities + where supertype = '{}';""".format(commodity_supertype)): + new_commodity_id = row_c[0] + # new_units = row_c[1] + for day_before, availability in schedule_dict[schedule_id]: + main_db_con.execute("""insert or ignore into vertices ( location_id, facility_id, + facility_type_id, facility_name, schedule_day, commodity_id, activity_level, + storage_vertex, demand, udp, source_facility_id, iob) values ({}, {}, {}, '{}', {}, {}, + {}, {}, {}, {}, {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, + facility_name, + day_before+1, new_commodity_id, availability, primary, + quantity, + the_scenario.unMetDemandPenalty, + zero_source_facility_id, iob)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, + activity_level, storage_vertex, demand, + source_facility_id, iob) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {}, '{}');""".format(facility_location_id, facility_id, facility_type_id, facility_name, + day_before+1, new_commodity_id, storage_availability, storage, quantity, + source_facility_id, iob)) + + else: + logger.warning( + "error, unexpected facility_type: {}, facility_type_id: {}".format(facility_type, facility_type_id)) + + for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): + logger.info('{} vertices created'.format(row_d[0])) + + logger.debug("total possible production in scenario: {}".format(total_potential_production)) + + +# =============================================================================== + + +def add_storage_routes(the_scenario, logger): + logger.info("start: add_storage_routes") + # these are loops to and from the same facility; when multiplied to edges, + # they will connect primary to storage vertices, and storage vertices day to day + # will always create edge for this route from storage to storage vertex + # IF a primary vertex exists, will also create an edge connecting the storage vertex to the primary + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + logger.debug("create the storage_routes table") + + main_db_con.execute("drop table if exists storage_routes;") + main_db_con.execute("""create table if not exists storage_routes as + select facility_name || '_storage' as route_name, + location_id, + facility_id, + facility_name as o_name, + facility_name as d_name, + {} as cost_1, + {} as cost_2, + 1 as travel_time, + {} as storage_max, + 0 as storage_min + from facilities + where ignore_facility = 'false' + ;""".format(storage_cost_1, storage_cost_2, facility_onsite_storage_max)) + + # drop and create route_reference table + # remove "drop" and replace with "create table if not exists" for cache + main_db_con.execute("drop table if exists route_reference;") + main_db_con.execute("""create table if not exists route_reference( + route_id INTEGER PRIMARY KEY, route_type text, route_name text, scenario_rt_id integer, from_node_id integer, + to_node_id integer, from_location_id integer, to_location_id integer, from_facility_id integer, to_facility_id integer, + commodity_id integer, phase_of_matter text, cost numeric, miles numeric, first_nx_edge_id integer, last_nx_edge_id integer, dollar_cost numeric, + CONSTRAINT unique_routes UNIQUE(route_type, route_name, scenario_rt_id));""") + main_db_con.execute( + "insert or ignore into route_reference(route_type, route_name, scenario_rt_id) select 'storage', route_name, 0 from storage" + "_routes;") + + return + + +# =============================================================================== + + +def generate_connector_and_storage_edges(the_scenario, logger): + logger.info("START: generate_connector_and_storage_edges") + + multi_commodity_name = "multicommodity" + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + if ('pipeline_crude_trf_rts' in the_scenario.permittedModes) or ( + 'pipeline_prod_trf_rts' in the_scenario.permittedModes): + logger.info("create indices for the capacity_nodes and pipeline_mapping tables") + main_db_con.executescript( + """ + CREATE INDEX IF NOT EXISTS pm_index + ON pipeline_mapping (id, id_field_name, mapping_id_field_name, mapping_id); + CREATE INDEX IF NOT EXISTS cn_index ON capacity_nodes (source, id_field_name, source_OID); + """) + + for row in db_cur.execute( + "select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): + id_for_mult_commodities = row[0] + + # create storage & connector edges + main_db_con.execute("drop table if exists edges;") + main_db_con.executescript(""" + create table edges (edge_id INTEGER PRIMARY KEY, + route_id integer, + from_node_id integer, + to_node_id integer, + start_day integer, + end_day integer, + commodity_id integer, + o_vertex_id integer, + d_vertex_id integer, + max_edge_capacity numeric, + volume numeric, + capac_minus_volume_zero_floor numeric, + min_edge_capacity numeric, + capacity_units text, + units_conversion_multiplier numeric, + edge_flow_cost numeric, + edge_flow_cost2 numeric, + edge_type text, + nx_edge_id integer, + mode text, + mode_oid integer, + miles numeric, + simple_mode text, + tariff_id numeric, + phase_of_matter text, + source_facility_id integer, + miles_travelled numeric, + children_created text, + edge_count_from_source integer, + total_route_cost numeric, + CONSTRAINT unique_nx_subc_day UNIQUE(nx_edge_id, commodity_id, source_facility_id, start_day)) + ; + + insert or ignore into edges (route_id, + start_day, end_day, + commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, + edge_flow_cost, edge_flow_cost2,edge_type, + source_facility_id) + select o.route_id, o.schedule_day, d.schedule_day, + o.commodity_id, o.vertex_id, d.vertex_id, + o.storage_max, o.storage_min, + o.cost_1, o.cost_2, 'storage', + o.source_facility_id + from vertices d, + (select v.vertex_id, v.schedule_day, + v.commodity_id, v.storage_vertex, v.source_facility_id, + t.* + from vertices v, + (select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, + storage_max, storage_min, rr.route_id, location_id, facility_id + from storage_routes sr, route_reference rr where sr.route_name = rr.route_name) t + where v.facility_id = t.facility_id + and v.storage_vertex = 1) o + where d.facility_id = o.facility_id + and d.schedule_day = o.schedule_day+o.travel_time + and d.commodity_id = o.commodity_id + and o.vertex_id != d.vertex_id + and d.storage_vertex = 1 + and d.source_facility_id = o.source_facility_id + ; + + insert or ignore into edges (route_id, start_day, end_day, + commodity_id, o_vertex_id, d_vertex_id, + edge_flow_cost, edge_type, + source_facility_id) + select s.route_id, s.schedule_day, p.schedule_day, + (case when s.commodity_id = {} then p.commodity_id else s.commodity_id end) commodity_id, + --inbound commodies start at storage and go into primary + --outbound starts at primary and goes into storage + --anything else is an error for a connector edge + (case when fc.io = 'i' then s.vertex_id + when fc.io = 'o' then p.vertex_id + else 0 end) as o_vertex, + (case when fc.io = 'i' then p.vertex_id + when fc.io = 'o' then s.vertex_id + else 0 end) as d_vertex, + 0, 'connector', + s.source_facility_id + from vertices p, facility_commodities fc, + --s for storage vertex info, p for primary vertex info + (select v.vertex_id, v.schedule_day, + v.commodity_id, v.storage_vertex, v.source_facility_id, + t.* + from vertices v, + (select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, + storage_max, storage_min, rr.route_id, location_id, facility_id + from storage_routes sr, route_reference rr where sr.route_name = rr.route_name) t --t is route data + where v.facility_id = t.facility_id + and v.storage_vertex = 1) s + --from storage into primary, same day = inbound connectors + where p.facility_id = s.facility_id + and p.schedule_day = s.schedule_day + and (p.commodity_id = s.commodity_id or p.commodity_id = {} ) + and p.facility_id = fc.facility_id + and fc.commodity_id = s.commodity_id + and p.storage_vertex = 0 + --either edge is inbound and aggregating, or kept separate by source, or primary vertex is not source tracked + and + (p.source_facility_id = 0 or p.source_facility_id = p.facility_id or p.source_facility_id = s.source_facility_id) + ;""".format(id_for_mult_commodities, id_for_mult_commodities)) + + for row_d in db_cur.execute("select count(distinct edge_id) from edges where edge_type = 'connector';"): + logger.info('{} connector edges created'.format(row_d[0])) + # clear any transport edges from table + db_cur.execute("delete from edges where edge_type = 'transport';") + + return + + +# =============================================================================== + + +def generate_first_edges_from_source_facilities(the_scenario, schedule_length, logger): + + logger.info("START: generate_first_edges_from_source_facilities") + # create edges table + # plan to generate start and end days based on nx edge time to traverse and schedule + # can still have route_id, but only for storage routes now; nullable + + # multi_commodity_name = "multicommodity" + transport_edges_created = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + edges_requiring_children = 0 + + counter = 0 + + # create transport edges, only between storage vertices and nodes, based on networkx graph + + commodity_mode_data = main_db_con.execute("select * from commodity_mode;") + commodity_mode_data = commodity_mode_data.fetchall() + commodity_mode_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + commodity_phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + commodity_mode_dict[mode, commodity_id] = allowed_yn + + + source_edge_data = main_db_con.execute("""select + ne.edge_id, + ifnull(CAST(fn.location_id as integer), 'NULL'), + ifnull(CAST(tn.location_id as integer), 'NULL'), + ne.mode_source, + ifnull(nec.phase_of_matter_id, 'NULL'), + nec.route_cost, + ne.from_node_id, + ne.to_node_id, + nec.dollar_cost, + ne.miles, + ne.capacity, + ne.artificial, + ne.mode_source_oid, + v.commodity_id, + v.schedule_day, + v.vertex_id, + v.source_facility_id, + tv.vertex_id, + ifnull(t.miles_travelled, 0), + ifnull(t.edge_count_from_source, 0), + t.mode + from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, vertices v, + facility_type_id ft, facility_commodities fc + left outer join (--get facilities with incoming transport edges with tracked mileage + select vi.facility_id, min(ei.miles_travelled) miles_travelled, ei.source_facility_id, + ei.edge_count_from_source, ei.mode + from edges ei, vertices vi + where vi.vertex_id = ei.d_vertex_id + and edge_type = 'transport' + and ifnull(miles_travelled, 0) > 0 + group by vi.facility_id, ei.source_facility_id, ei.mode) t + on t.facility_id = v.facility_id and t.source_facility_id = v.source_facility_id and ne.mode_source = t.mode + left outer join vertices tv + on (CAST(tn.location_id as integer) = tv.location_id and v.source_facility_id = tv.source_facility_id) + where v.location_id = CAST(fn.location_id as integer) + and fc.facility_id = v.facility_id + and fc.commodity_id = v.commodity_id + and fc.io = 'o' + and ft.facility_type_id = v.facility_type_id + and (ft.facility_type = 'raw_material_producer' or t.facility_id = v.facility_id) + and ne.from_node_id = fn.node_id + and ne.to_node_id = tn.node_id + and ne.edge_id = nec.edge_id + and ifnull(ne.capacity, 1) > 0 + and v.storage_vertex = 1 + and v.source_facility_id != 0 --max commodity distance applies + ;""") + source_edge_data = source_edge_data.fetchall() + for row_a in source_edge_data: + + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + # max_daily_capacity = row_a[10] + # artificial = row_a[11] + mode_oid = row_a[12] + commodity_id = row_a[13] + origin_day = row_a[14] + vertex_id = row_a[15] + source_facility_id = row_a[16] + to_vertex = row_a[17] + previous_miles_travelled = row_a[18] + previous_edge_count = row_a[19] + previous_mode = row_a[20] + + simple_mode = row_a[3].partition('_')[0] + edge_count_from_source = 1 + previous_edge_count + total_route_cost = route_cost + miles_travelled = previous_miles_travelled + miles + + if counter % 10000 == 0: + for row_d in db_cur.execute("select count(distinct edge_id) from edges;"): + logger.info('{} edges created'.format(row_d[0])) + counter = counter + 1 + + tariff_id = 0 + if simple_mode == 'pipeline': + + # find tariff_ids + + sql = "select mapping_id from pipeline_mapping " \ + "where id = {} and id_field_name = 'source_OID' " \ + "and source = '{}' and mapping_id is not null;".format( + mode_oid, mode) + for tariff_row in db_cur.execute(sql): + tariff_id = tariff_row[0] + + + if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys() \ + and commodity_mode_dict[mode, commodity_id] == 'Y': + + # Edges are placeholders for flow variables + # 4-17: if both ends have no location, iterate through viable commodities and days, create edge + # for all days (restrict by link schedule if called for) + # for all allowed commodities, as currently defined by link phase of matter + + # days range from 1 to schedule_length + if origin_day in range(1, schedule_length+1): + if origin_day + fixed_route_duration <= schedule_length: + # if link is traversable in the timeframe + if simple_mode != 'pipeline' or tariff_id >= 0: + # for allowed commodities + # step 1 from source is from non-Null location to (probably) null location + + if from_location != 'NULL' and to_location == 'NULL': + # for each day and commodity, + # get the corresponding origin vertex id to include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough + # than checking if facility type is 'ultimate destination' + # only connect to vertices with matching source_facility_id + # source_facility_id is zero for commodities without source tracking + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id,phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + miles_travelled, 'N', edge_count_from_source, total_route_cost)) + + + elif from_location != 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding origin and destination vertex ids + # to include with the edge info + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, {}, + {}, {}, {}, + '{}',{},'{}', {}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, to_vertex, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + miles_travelled, 'N', edge_count_from_source, total_route_cost)) + + for row in db_cur.execute("select count(distinct edge_id) from edges where edge_type = 'transport';"): + transport_edges_created = row[0] + logger.info('{} transport edges created'.format(transport_edges_created)) + for row in db_cur.execute("""select count(distinct edge_id), children_created from edges + where edge_type = 'transport' + group by children_created;"""): + if row[1] == 'N': + edges_requiring_children = row[0] + + elif row[1] == 'Y': + logger.info('{} transport edges that have already been checked for children'.format(row[0])) + edges_requiring_children = transport_edges_created - row[0] + # edges_requiring_children is updated under either condition here, since one may not be triggered + logger.info('{} transport edges that need children'.format(edges_requiring_children)) + + return + + +# =============================================================================== + + +def generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger): + # method only runs for commodities with a max commodity constraint + + logger.info("START: generate_all_edges_from_source_facilities") + + multi_commodity_name = "multicommodity" + # initializations - all of these get updated if >0 edges exist + edges_requiring_children = 0 + endcap_edges = 0 + edges_resolved = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + transport_edges_created = 0 + nx_edge_count = 0 + source_based_edges_created = 0 + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport' + and children_created in ('N', 'Y', 'E');"""): + source_based_edges_created = row_d[0] + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport';"""): + transport_edges_created = row_d[0] + nx_edge_count = row_d[1] + + + commodity_mode_data = main_db_con.execute("select * from commodity_mode;") + commodity_mode_data = commodity_mode_data.fetchall() + commodity_mode_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + commodity_phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + commodity_mode_dict[mode, commodity_id] = allowed_yn + + current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created + from edges + where edge_type = 'transport' + group by children_created + order by children_created asc;""") + current_edge_data = current_edge_data.fetchall() + for row in current_edge_data: + if row[1] == 'N': + edges_requiring_children = row[0] + elif row[1] == 'Y': + edges_resolved = row[0] + elif row[1] == 'E': + endcap_edges = row[0] + if source_based_edges_created == edges_resolved + endcap_edges: + edges_requiring_children = 0 + if source_based_edges_created == edges_requiring_children + endcap_edges: + edges_resolved = 0 + if source_based_edges_created == edges_requiring_children + edges_resolved: + endcap_edges = 0 + + logger.info( + '{} transport edges created; {} require children'.format(transport_edges_created, edges_requiring_children)) + + # set up a table to keep track of endcap nodes + sql = """ + drop table if exists endcap_nodes; + create table if not exists endcap_nodes( + node_id integer NOT NULL, + + location_id integer, + + --mode of edges that it's an endcap for + mode_source text NOT NULL, + + --facility it's an endcap for + source_facility_id integer NOT NULL, + + --commodities it's an endcap for + commodity_id integer NOT NULL, + + CONSTRAINT endcap_key PRIMARY KEY (node_id, mode_source, source_facility_id, commodity_id)) + --the combination of these four (excluding location_id) should be unique, + --and all fields except location_id should be filled + ;""" + db_cur.executescript(sql) + + # create transport edges, only between storage vertices and nodes, based on networkx graph + + edge_into_facility_counter = 0 + while_count = 0 + + while edges_requiring_children > 0: + while_count = while_count+1 + + # --get nx edges that align with the existing "in edges" - data from nx to create new edges --for each of + # those nx_edges, if they connect to more than one "in edge" in this batch, only consider connecting to + # the shortest -- if there is a valid nx_edge to build, the crossing node is not an endcap if total miles + # of the new route is over max transport distance, then endcap what if one child goes over max transport + # and another doesn't then the node will get flagged as an endcap, and another path may continue off it, + # allow both for now --check that day and commodity are permitted by nx + + potential_edge_data = main_db_con.execute(""" + select + ch.edge_id as ch_nx_edge_id, + ifnull(CAST(chfn.location_id as integer), 'NULL') fn_location_id, + ifnull(CAST(chtn.location_id as integer), 'NULL') tn_location_id, + ch.mode_source, + p.phase_of_matter, + nec.route_cost, + ch.from_node_id, + ch.to_node_id, + nec.dollar_cost, + ch.miles, + ch.capacity, + ch.artificial, + ch.mode_source_oid, + --parent edge into + p.commodity_id, + p.end_day, + --parent's dest. vertex if exists + ifnull(p.d_vertex_id,0) o_vertex, + p.source_facility_id, + p.leadin_miles_travelled, + + (p.edge_count_from_source +1) as new_edge_count, + (p.total_route_cost + nec.route_cost) new_total_route_cost, + p.edge_id leadin_edge, + p.nx_edge_id leadin_nx_edge, + + --new destination vertex if exists + ifnull(chtv.vertex_id,0) d_vertex, + + sc.max_transport_distance + + from + (select count(edge_id) parents, + min(miles_travelled) leadin_miles_travelled, + * + from edges + where children_created = 'N' + -----------------do not mess with this "group by" + group by to_node_id, source_facility_id, commodity_id, end_day + ------------------it affects which columns we're checking over for min miles travelled + --------------so that we only get the parent edges we want + order by parents desc + ) p, --parent edges to use in this batch + networkx_edges ch, + networkx_edge_costs nec, + source_commodity_ref sc, + networkx_nodes chfn, + networkx_nodes chtn + left outer join vertices chtv + on (CAST(chtn.location_id as integer) = chtv.location_id + and p.source_facility_id = chtv.source_facility_id + and chtv.commodity_id = p.commodity_id + and p.end_day = chtv.schedule_day + and chtv.iob = 'i') + + where p.to_node_id = ch.from_node_id + --and p.mode = ch.mode_source --build across modes, control at conservation of flow + and ch.to_node_id = chtn.node_id + and ch.from_node_id = chfn.node_id + and p.phase_of_matter = nec.phase_of_matter_id + and ch.edge_id = nec.edge_id + and ifnull(ch.capacity, 1) > 0 + and p.commodity_id = sc.commodity_id + ;""") + + # --should only get a single leadin edge per networkx/source/commodity/day combination + # leadin edge should be in route_data, current set of min. identifiers + # if we're trying to add an edge that has an entry in route_data, new miles travelled must be less + + potential_edge_data = potential_edge_data.fetchall() + + main_db_con.execute("update edges set children_created = 'Y' where children_created = 'N';") + + for row_a in potential_edge_data: + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + mode_oid = row_a[12] + commodity_id = row_a[13] + origin_day = row_a[14] + vertex_id = row_a[15] + source_facility_id = row_a[16] + leadin_edge_miles_travelled = row_a[17] + new_edge_count = row_a[18] + total_route_cost = row_a[19] + leadin_edge_id = row_a[20] + # leadin_nx_edge_id = row_a[21] + to_vertex = row_a[22] + max_commodity_travel_distance = row_a[23] + + # end_day = origin_day + fixed_route_duration + new_miles_travelled = miles + leadin_edge_miles_travelled + + if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys() \ + and commodity_mode_dict[mode, commodity_id] == 'Y': + + if new_miles_travelled > max_commodity_travel_distance: + # designate leadin edge as endcap + children_created = 'E' + # update the incoming edge to indicate it's an endcap + db_cur.execute( + "update edges set children_created = '{}' where edge_id = {}".format(children_created, + leadin_edge_id)) + if from_location != 'NULL': + db_cur.execute("""insert or ignore into endcap_nodes( + node_id, location_id, mode_source, source_facility_id, commodity_id) + VALUES ({}, {}, '{}', {}, {}); + """.format(from_node, from_location, mode, source_facility_id, commodity_id)) + else: + db_cur.execute("""insert or ignore into endcap_nodes( + node_id, mode_source, source_facility_id, commodity_id) + VALUES ({}, '{}', {}, {}); + """.format(from_node, mode, source_facility_id, commodity_id)) + + # create new edge + elif new_miles_travelled <= max_commodity_travel_distance: + + simple_mode = row_a[3].partition('_')[0] + tariff_id = 0 + if simple_mode == 'pipeline': + + # find tariff_ids + + sql = """select mapping_id + from pipeline_mapping + where id = {} + and id_field_name = 'source_OID' + and source = '{}' + and mapping_id is not null;""".format(mode_oid, mode) + for tariff_row in db_cur.execute(sql): + tariff_id = tariff_row[0] + + # if there are no edges yet for this day, nx, subc combination, + # AND this is the shortest existing leadin option for this day, nx, subc combination + # we'd be creating an edge for (otherwise wait for the shortest option) + # at this step, some leadin edge should always exist + + if origin_day in range(1, schedule_length+1): + if origin_day + fixed_route_duration <= schedule_length: + # if link is traversable in the timeframe + if simple_mode != 'pipeline' or tariff_id >= 0: + # for allowed commodities + + if from_location == 'NULL' and to_location == 'NULL': + # for each day and commodity, + # get the corresponding origin vertex id to include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough than + # checking if facility type is 'ultimate destination' + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id,phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, {}, {}, + '{}',{},'{}',{}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + # only create edge going into a location if an appropriate vertex exists + elif from_location == 'NULL' and to_location != 'NULL' and to_vertex > 0: + edge_into_facility_counter = edge_into_facility_counter + 1 + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}', {}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + to_vertex, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + # designate leadin edge as endcap + # this does, deliberately, allow endcap status to be + # overwritten if we've found a shorter path to a previous endcap + elif from_location != 'NULL' and to_location == 'NULL': + # for each day and commodity, get the corresponding origin vertex id + # to include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough than + # checking if facility type is 'ultimate destination' + # new for bsc, only connect to vertices with matching source facility id + # (only limited for RMP vertices) + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + + elif from_location != 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding origin and + # destination vertex ids to include with the edge info + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, + total_route_cost) + VALUES ({}, {}, + {}, {}, {}, + {}, {}, + {}, {}, {}, + '{}',{},'{}', {}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, to_vertex, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport' + and children_created in ('N', 'Y', 'E');"""): + source_based_edges_created = row_d[0] + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport';"""): + transport_edges_created = row_d[0] + nx_edge_count = row_d[1] + + current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created + from edges + where edge_type = 'transport' + group by children_created + order by children_created asc;""") + current_edge_data = current_edge_data.fetchall() + for row in current_edge_data: + if row[1] == 'N': + edges_requiring_children = row[0] + elif row[1] == 'Y': + edges_resolved = row[0] + elif row[1] == 'E': + endcap_edges = row[0] + logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) + if source_based_edges_created == edges_resolved + endcap_edges: + edges_requiring_children = 0 + if source_based_edges_created == edges_requiring_children + endcap_edges: + edges_resolved = 0 + if source_based_edges_created == edges_requiring_children + edges_resolved: + endcap_edges = 0 + + if while_count % 1000 == 0 or edges_requiring_children == 0: + logger.info( + '{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( + transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) + + # edges going in to the facility by re-running "generate first edges + # then re-run this method + + logger.info('{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( + transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) + logger.info("all source-based transport edges created") + + logger.info("create an index for the edges table by nodes") + + sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( + edge_id, route_id, from_node_id, to_node_id, commodity_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") + db_cur.execute(sql) + + return + + +# =============================================================================== + + +def generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger): + + global total_transport_routes + logger.info("START: generate_all_edges_without_max_commodity_constraint") + + multi_commodity_name = "multicommodity" + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + + commodity_mode_data = main_db_con.execute("select * from commodity_mode;") + commodity_mode_data = commodity_mode_data.fetchall() + commodity_mode_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + commodity_phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + commodity_mode_dict[mode, commodity_id] = allowed_yn + + counter = 0 + # for row in db_cur.execute( + # "select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): + # id_for_mult_commodities = row[0] + logger.info("COUNTING TOTAL TRANSPORT ROUTES") + for row in db_cur.execute(""" + select count(*) from networkx_edges, shortest_edges + WHERE networkx_edges.from_node_id = shortest_edges.from_node_id + AND networkx_edges.to_node_id = shortest_edges.to_node_id + AND networkx_edges.edge_id = shortest_edges.edge_id + ; + """): + total_transport_routes = row[0] + + # for all commodities with no max transport distance + source_facility_id = 0 + + # create transport edges, only between storage vertices and nodes, based on networkx graph + # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link + # iterate through nx edges: if neither node has a location, create 1 edge per viable commodity + # should also be per day, subject to nx edge schedule + # before creating an edge, check: commodity allowed by nx and max transport distance if not null + # will need nodes per day and commodity? or can I just check that with constraints? + # select data for transport edges + + sql = """select + ne.edge_id, + ifnull(fn.location_id, 'NULL'), + ifnull(tn.location_id, 'NULL'), + ne.mode_source, + ifnull(nec.phase_of_matter_id, 'NULL'), + nec.route_cost, + ne.from_node_id, + ne.to_node_id, + nec.dollar_cost, + ne.miles, + ne.capacity, + ne.artificial, + ne.mode_source_oid + from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, shortest_edges se + + where ne.from_node_id = fn.node_id + and ne.to_node_id = tn.node_id + and ne.edge_id = nec.edge_id + and ne.from_node_id = se.from_node_id -- match to the shortest_edges table + and ne.to_node_id = se.to_node_id + and ifnull(ne.capacity, 1) > 0 + ;""" + nx_edge_data = main_db_con.execute(sql) + nx_edge_data = nx_edge_data.fetchall() + for row_a in nx_edge_data: + + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + # max_daily_capacity = row_a[10] + mode_oid = row_a[12] + simple_mode = row_a[3].partition('_')[0] + + counter = counter + 1 + + tariff_id = 0 + if simple_mode == 'pipeline': + + # find tariff_ids + + sql = "select mapping_id from pipeline_mapping " \ + "where id = {} and id_field_name = 'source_OID' and source = '{}' " \ + "and mapping_id is not null;".format( + mode_oid, mode) + for tariff_row in db_cur.execute(sql): + tariff_id = tariff_row[0] + + + if mode in the_scenario.permittedModes: + + # Edges are placeholders for flow variables + # for all days (restrict by link schedule if called for) + # for all allowed commodities, as currently defined by link phase of matter + + for day in range(1, schedule_length+1): + if day + fixed_route_duration <= schedule_length: + # if link is traversable in the timeframe + if simple_mode != 'pipeline' or tariff_id >= 0: + # for allowed commodities that can be output by some facility in the scenario + for row_c in db_cur.execute("""select commodity_id + from source_commodity_ref s + where phase_of_matter = '{}' + and max_transport_distance_flag = 'N' + and share_max_transport_distance = 'N' + group by commodity_id, source_facility_id""".format(phase_of_matter)): + db_cur4 = main_db_con.cursor() + commodity_id = row_c[0] + # source_facility_id = row_c[1] # fixed to 0 for all edges created by this method + if (mode, commodity_id) in commodity_mode_dict.keys() \ + and commodity_mode_dict[mode, commodity_id] == 'Y': + if from_location == 'NULL' and to_location == 'NULL': + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + + elif from_location != 'NULL' and to_location == 'NULL': + # for each day and commodity, get the corresponding origin vertex id + # to include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough + # than checking if facility type is 'ultimate destination' + # new for bsc, only connect to vertices withr matching source_facility_id + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): + from_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + from_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + elif from_location == 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding destination vertex id + # to include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): + to_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + to_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + elif from_location != 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding origin and destination vertex + # ids to include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): + from_vertex_id = row_d[0] + db_cur5 = main_db_con.cursor() + for row_e in db_cur5.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): + to_vertex_id = row_e[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, + to_node_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, edge_type, + nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, {}, {}, {}, {}, {}, + {}, {}, {}, '{}',{},'{}', {},{},'{}',{},'{}',{} + )""".format(from_node, + to_node, day, + day + fixed_route_duration, + commodity_id, + from_vertex_id, + to_vertex_id, + default_min_capacity, + route_cost, + dollar_cost, + 'transport', + nx_edge_id, mode, + mode_oid, miles, + simple_mode, + tariff_id, + phase_of_matter, + source_facility_id)) + + logger.debug("all transport edges created") + + logger.info("all edges created") + logger.info("create an index for the edges table by nodes") + index_start_time = datetime.datetime.now() + sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( + edge_id, route_id, from_node_id, to_node_id, commodity_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") + db_cur.execute(sql) + logger.info("edge_index Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(index_start_time))) + return + + +# =============================================================================== + + +def generate_edges_from_routes(the_scenario, schedule_length, logger): + # ROUTES - create a transport edge for each route by commodity, day, etc. + logger.info("START: generate_edges_from_routes") + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + + # Filter edges table on those in shortest edges, group by + # phase, commodity, from, and to locations to populate a routes table + # db_cur.execute("""select distinct from_location_id, to_location_id, + # from shortest_edges se + # """) + + # From Olivia + db_cur.execute("""insert or ignore into route_reference (route_type,scenario_rt_id,from_node_id,to_node_id,from_location_id,to_location_id,from_facility_id,to_facility_id,cost,miles,phase_of_matter,commodity_id,first_nx_edge_id,last_nx_edge_id,dollar_cost) + select 'transport', odp.scenario_rt_id, odp.from_node_id, odp.to_node_id,odp.from_location_id,odp.to_location_id, + odp.from_facility_id, odp.to_facility_id, r2.cost, + r2.miles, odp.phase_of_matter, odp.commodity_id, r2.first_nx_edge, r2.last_nx_edge, r2.dollar_cost + FROM od_pairs odp, + (select r1.scenario_rt_id, r1.miles, r1.cost, r1.dollar_cost, r1.num_edges, re1.edge_id as first_nx_edge, re2.edge_id as last_nx_edge, r1.phase_of_matter from + (select scenario_rt_id, sum(e.miles) as miles, sum(e.route_cost) as cost, sum(e.dollar_cost) as dollar_cost, max(rt_order_ind) as num_edges, e.phase_of_matter_id as phase_of_matter --, count(e.edge_id) + from route_edges re + LEFT OUTER JOIN --everything from the route edges table, only edge data from the adhoc table that matches route_id + (select ne.edge_id, + nec.route_cost as route_cost, + nec.dollar_cost as dollar_cost, + ne.miles as miles, + ne.mode_source as mode, + nec.phase_of_matter_id + from networkx_edges ne, networkx_edge_costs nec --or Edges table? + where nec.edge_id = ne.edge_id) e --this is the adhoc edge info table + on re.edge_id=e.edge_id + group by scenario_rt_id, phase_of_matter) r1 + join (select * from route_edges where rt_order_ind = 1) re1 on r1.scenario_rt_id = re1.scenario_rt_id + join route_edges re2 on r1.scenario_rt_id = re2.scenario_rt_id and r1.num_edges = re2.rt_order_ind) r2 + where r2.scenario_rt_id = odp.scenario_rt_id and r2.phase_of_matter = odp.phase_of_matter + ;""") + + route_data = main_db_con.execute("select * from route_reference where route_type = 'transport';") + + # Add an edge for each route, (applicable) vertex, day, commodity + for row_a in route_data: + route_id = row_a[0] + from_node_id = row_a[4] + to_node_id = row_a[5] + from_location = row_a[6] + to_location = row_a[7] + commodity_id = row_a[10] + phase_of_matter = row_a[11] + cost = row_a[12] + miles = row_a[13] + source_facility_id = 0 + + for day in range(1, schedule_length+1): + if day + fixed_route_duration <= schedule_length: + # add edge from o_vertex to d_vertex + # for each day and commodity, get the corresponding origin and destination vertex + # ids to include with the edge info + db_cur4 = main_db_con.cursor() + # TODO: are these DB calls necessary for vertices? + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): + from_vertex_id = row_d[0] + db_cur5 = main_db_con.cursor() + for row_e in db_cur5.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} + and v.commodity_id = {} and v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): + to_vertex_id = row_e[0] + main_db_con.execute("""insert or ignore into edges (route_id, from_node_id, + to_node_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + edge_flow_cost, edge_type, + miles,phase_of_matter) VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, + '{}', {},'{}' + )""".format(route_id, + from_node_id, + to_node_id, + day, + day + fixed_route_duration, + commodity_id, + from_vertex_id, to_vertex_id, + cost, + 'transport', + # nx_edge_id, mode, mode_oid, + miles, + # simple_mode, tariff_id, + phase_of_matter)) + #source_facility_id)) + + return + + +# =============================================================================== + + +def set_edges_volume_capacity(the_scenario, logger): + logger.info("starting set_edges_volume_capacity") + with sqlite3.connect(the_scenario.main_db) as main_db_con: + logger.debug("starting to record volume and capacity for non-pipeline edges") + + main_db_con.execute( + "update edges set volume = (select ifnull(ne.volume,0) from networkx_edges ne " + "where ne.edge_id = edges.nx_edge_id ) where simple_mode in ('rail','road','water');") + main_db_con.execute( + "update edges set max_edge_capacity = (select ne.capacity from networkx_edges ne " + "where ne.edge_id = edges.nx_edge_id) where simple_mode in ('rail','road','water');") + logger.debug("volume and capacity recorded for non-pipeline edges") + + logger.debug("starting to record volume and capacity for pipeline edges") + ## + main_db_con.executescript("""update edges set volume = + (select l.background_flow + from pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where edges.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(edges.mode, l.source)>0) + where simple_mode = 'pipeline' + ; + + update edges set max_edge_capacity = + (select l.capac + from pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where edges.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(edges.mode, l.source)>0) + where simple_mode = 'pipeline' + ;""") + logger.debug("volume and capacity recorded for pipeline edges") + logger.debug("starting to record units and conversion multiplier") + main_db_con.execute("""update edges + set capacity_units = + (case when simple_mode = 'pipeline' then 'kbarrels' + when simple_mode = 'road' then 'truckload' + when simple_mode = 'rail' then 'railcar' + when simple_mode = 'water' then 'barge' + else 'unexpected mode' end) + ;""") + main_db_con.execute("""update edges + set units_conversion_multiplier = + (case when simple_mode = 'pipeline' and phase_of_matter = 'liquid' then {} + when simple_mode = 'road' and phase_of_matter = 'liquid' then {} + when simple_mode = 'road' and phase_of_matter = 'solid' then {} + when simple_mode = 'rail' and phase_of_matter = 'liquid' then {} + when simple_mode = 'rail' and phase_of_matter = 'solid' then {} + when simple_mode = 'water' and phase_of_matter = 'liquid' then {} + when simple_mode = 'water' and phase_of_matter = 'solid' then {} + else 1 end) + ;""".format(THOUSAND_GALLONS_PER_THOUSAND_BARRELS, + the_scenario.truck_load_liquid.magnitude, + the_scenario.truck_load_solid.magnitude, + the_scenario.railcar_load_liquid.magnitude, + the_scenario.railcar_load_solid.magnitude, + the_scenario.barge_load_liquid.magnitude, + the_scenario.barge_load_solid.magnitude, + )) + logger.debug("units and conversion multiplier recorded for all edges; starting capacity minus volume") + main_db_con.execute("""update edges + set capac_minus_volume_zero_floor = + max((select (max_edge_capacity - ifnull(volume,0)) where max_edge_capacity is not null),0) + where max_edge_capacity is not null + ;""") + logger.debug("capacity minus volume (minimum set to zero) recorded for all edges") + return + + +# =============================================================================== + + +def pre_setup_pulp(logger, the_scenario): + logger.info("START: pre_setup_pulp") + + commodity_mode_setup(the_scenario, logger) + + # create table to track source facility of commodities with a max transport distance set + source_tracking_setup(the_scenario, logger) + + schedule_dict, schedule_length = generate_schedules(the_scenario, logger) + + generate_all_vertices(the_scenario, schedule_dict, schedule_length, logger) + + add_storage_routes(the_scenario, logger) + generate_connector_and_storage_edges(the_scenario, logger) + + if not the_scenario.ndrOn: + # start edges for commodities that inherit max transport distance + generate_first_edges_from_source_facilities(the_scenario, schedule_length, logger) + + # replicate all_routes by commodity and time into all_edges dictionary + generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger) + + # replicate all_routes by commodity and time into all_edges dictionary + generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger) + logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) + + else: + generate_edges_from_routes(the_scenario, schedule_length, logger) + + set_edges_volume_capacity(the_scenario, logger) + + return + + +# =============================================================================== + + +def create_flow_vars(the_scenario, logger): + logger.info("START: create_flow_vars") + + # we have a table called edges. + # call helper method to get list of unique IDs from the Edges table. + # use the rowid as a simple unique integer index + edge_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, source_facility_id + from edges;""") + edge_list_data = edge_list_cur.fetchall() + counter = 0 + for row in edge_list_data: + if counter % 500000 == 0: + logger.info( + "processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) + counter += 1 + # create an edge for each commodity allowed on this link - this construction may change + # as specific commodity restrictions are added + # TODO4-18 add days, but have no scheduel for links currently + # running just with nodes for now, will add proper facility info and storage back soon + edge_list.append((row[0])) + + + flow_var = LpVariable.dicts("Edge", edge_list, 0, None) + return flow_var + + +# =============================================================================== + + +def create_unmet_demand_vars(the_scenario, logger): + logger.info("START: create_unmet_demand_vars") + demand_var_list = [] + # may create vertices with zero demand, but only for commodities that the facility has demand for at some point + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day, + ifnull(c.supertype, c.commodity_name) top_level_commodity_name, v.udp + from vertices v, commodities c, facility_type_id ft, facilities f + where v.commodity_id = c.commodity_id + and ft.facility_type = "ultimate_destination" + and v.storage_vertex = 0 + and v.facility_type_id = ft.facility_type_id + and v.facility_id = f.facility_id + and f.ignore_facility = 'false' + group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) + ;""".format('')): + # facility_id, day, and simplified commodity name + demand_var_list.append((row[0], row[1], row[2], row[3])) + + unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) + + return unmet_demand_var + + +# =============================================================================== + + +def create_candidate_processor_build_vars(the_scenario, logger): + logger.info("START: create_candidate_processor_build_vars") + processors_build_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute( + """select f.facility_id from facilities f, facility_type_id ft + where f.facility_type_id = ft.facility_type_id and facility_type = 'processor' + and candidate = 1 and ignore_facility = 'false' group by facility_id;"""): + # grab all candidate processor facility IDs + processors_build_list.append(row[0]) + + processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list, 0, None, 'Binary') + + return processor_build_var + + +# =============================================================================== + + +def create_binary_processor_vertex_flow_vars(the_scenario, logger): + logger.info("START: create_binary_processor_vertex_flow_vars") + processors_flow_var_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 0 + group by v.facility_id, v.schedule_day;"""): + # facility_id, day + processors_flow_var_list.append((row[0], row[1])) + + processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0, None, 'Binary') + + return processor_flow_var + + +# =============================================================================== + + +def create_processor_excess_output_vars(the_scenario, logger): + logger.info("START: create_processor_excess_output_vars") + + excess_var_list = [] + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + xs_cur = db_cur.execute(""" + select vertex_id, commodity_id + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 1;""") + # facility_id, day, and simplified commodity name + xs_data = xs_cur.fetchall() + for row in xs_data: + excess_var_list.append(row[0]) + + excess_var = LpVariable.dicts("XS", excess_var_list, 0, None) + + return excess_var + + +# =============================================================================== + + +def create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars): + logger.debug("START: create_opt_problem") + prob = LpProblem("Flow assignment", LpMinimize) + + unmet_demand_costs = [] + flow_costs = {} + processor_build_costs = [] + for u in unmet_demand_vars: + # facility_id = u[0] + # schedule_day = u[1] + # demand_commodity_name = u[2] + udp = u[3] + unmet_demand_costs.append(udp * unmet_demand_vars[u]) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + # Flow cost memory improvements: only get needed data; dict instead of list; narrow in lpsum + flow_cost_var = db_cur.execute("select edge_id, edge_flow_cost from edges e group by edge_id;") + flow_cost_data = flow_cost_var.fetchall() + counter = 0 + for row in flow_cost_data: + edge_id = row[0] + edge_flow_cost = row[1] + counter += 1 + + # flow costs cover transportation and storage + flow_costs[edge_id] = edge_flow_cost + # flow_costs.append(edge_flow_cost * flow_vars[(edge_id)]) + + logger.info("check if candidate tables exist") + sql = "SELECT name FROM sqlite_master WHERE type='table' " \ + "AND name in ('candidate_processors', 'candidate_process_list');" + count = len(db_cur.execute(sql).fetchall()) + + if count == 2: + + processor_build_cost = db_cur.execute(""" + select f.facility_id, (p.cost_formula*c.quantity) build_cost + from facilities f, facility_type_id ft, candidate_processors c, candidate_process_list p + where f.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and candidate = 1 + and ignore_facility = 'false' + and f.facility_name = c.facility_name + and c.process_id = p.process_id + group by f.facility_id, build_cost;""") + processor_build_cost_data = processor_build_cost.fetchall() + for row in processor_build_cost_data: + candidate_proc_facility_id = row[0] + proc_facility_build_cost = row[1] + processor_build_costs.append( + proc_facility_build_cost * processor_build_vars[candidate_proc_facility_id]) + + prob += (lpSum(unmet_demand_costs) + lpSum(flow_costs[k] * flow_vars[k] for k in flow_costs) + lpSum( + processor_build_costs)), "Total Cost of Transport, storage, facility building, and penalties" + + logger.debug("FINISHED: create_opt_problem") + return prob + + +# =============================================================================== + + +def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): + logger.debug("START: create_constraint_unmet_demand") + + # apply activity_level to get corresponding actual demand for var + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + # var has form(facility_name, day, simple_fuel) + # unmet demand commodity should be simple_fuel = supertype + + demand_met_dict = defaultdict(list) + actual_demand_dict = {} + + # demand_met = [] + # want to specify that all edges leading into this vertex + unmet demand = total demand + # demand primary (non-storage) vertices + + db_cur = main_db_con.cursor() + # each row_a is a primary vertex whose edges in contributes to the met demand of var + # will have one row for each fuel subtype in the scenario + unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + from vertices v, commodities c, facility_type_id ft, facilities f, edges e + where v.facility_id = f.facility_id + and ft.facility_type = 'ultimate_destination' + and f.facility_type_id = ft.facility_type_id + and f.ignore_facility = 'false' + and v.facility_type_id = ft.facility_type_id + and v.storage_vertex = 0 + and c.commodity_id = v.commodity_id + and e.d_vertex_id = v.vertex_id + group by v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + ;""") + + unmet_data = unmet_data.fetchall() + for row_a in unmet_data: + # primary_vertex_id = row_a[0] + # commodity_id = row_a[1] + var_full_demand = row_a[2] + proportion_of_supertype = row_a[3] + var_activity_level = row_a[4] + # source_facility_id = row_a[5] + facility_id = row_a[6] + day = row_a[7] + top_level_commodity = row_a[8] + udp = row_a[9] + edge_id = row_a[10] + var_actual_demand = var_full_demand * var_activity_level + + # next get inbound edges, apply appropriate modifier proportion to get how much of var's demand they satisfy + demand_met_dict[(facility_id, day, top_level_commodity, udp)].append( + flow_var[edge_id] * proportion_of_supertype) + actual_demand_dict[(facility_id, day, top_level_commodity, udp)] = var_actual_demand + + for key in unmet_demand_var: + if key in demand_met_dict: + # then there are some edges in + prob += lpSum(demand_met_dict[key]) == actual_demand_dict[key] - unmet_demand_var[ + key], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(key[0], + key[1], + key[2]) + else: + if key not in actual_demand_dict: + pdb.set_trace() + # no edges in, so unmet demand equals full demand + prob += actual_demand_dict[key] == unmet_demand_var[ + key], "constraint set unmet demand variable for facility {}, day {}, " \ + "commodity {} - no edges able to meet demand".format( + key[0], key[1], key[2]) + + logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") + return prob + + +# =============================================================================== + + +def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + + # create_constraint_max_flow_out_of_supply_vertex + # primary vertices only + # flow out of a vertex <= supply of the vertex, true for every day and commodity + + # for each primary (non-storage) supply vertex + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row_a in db_cur.execute("""select vertex_id, activity_level, supply + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and ft.facility_type = 'raw_material_producer' + and storage_vertex = 0;"""): + supply_vertex_id = row_a[0] + activity_level = row_a[1] + max_daily_supply = row_a[2] + actual_vertex_supply = activity_level * max_daily_supply + + flow_out = [] + db_cur2 = main_db_con.cursor() + # select all edges leaving that vertex and sum their flows + # should be a single connector edge + for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): + edge_id = row_b[0] + flow_out.append(flow_var[edge_id]) + + prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format( + actual_vertex_supply, supply_vertex_id) + # could easily add human-readable vertex info to this if desirable + + logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") + return prob + + +# =============================================================================== + + +def create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_var, processor_build_vars, + processor_daily_flow_vars): + logger.debug("STARTING: create_constraint_daily_processor_capacity") + import pdb + # pdb.set_trace() + # primary vertices only + # flow into vertex is capped at facility max_capacity per day + # sum over all input commodities, grouped by day and facility + # conservation of flow and ratios are handled in other methods + + ### get primary processor vertex and its input quantityi + total_scenario_min_capacity = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + sql = """select f.facility_id, + ifnull(f.candidate, 0), ifnull(f.max_capacity, -1), v.schedule_day, v.activity_level + from facility_commodities fc, facility_type_id ft, facilities f, vertices v + where ft.facility_type = 'processor' + and ft.facility_type_id = f.facility_type_id + and f.facility_id = fc.facility_id + and fc.io = 'i' + and v.facility_id = f.facility_id + and v.storage_vertex = 0 + group by f.facility_id, ifnull(f.candidate, 0), f.max_capacity, v.schedule_day, v.activity_level + ; + """ + # iterate through processor facilities, one constraint per facility per day + # no handling of subcommodities + + processor_facilities = db_cur.execute(sql) + + processor_facilities = processor_facilities.fetchall() + + for row_a in processor_facilities: + + # input_commodity_id = row_a[0] + facility_id = row_a[0] + is_candidate = row_a[1] + max_capacity = row_a[2] + day = row_a[3] + daily_activity_level = row_a[4] + + if max_capacity >= 0: + daily_inflow_max_capacity = float(max_capacity) * float(daily_activity_level) + daily_inflow_min_capacity = daily_inflow_max_capacity / 2 + logger.debug( + "processor {}, day {}, input capacity min: {} max: {}".format(facility_id, day, daily_inflow_min_capacity, + daily_inflow_max_capacity)) + total_scenario_min_capacity = total_scenario_min_capacity + daily_inflow_min_capacity + flow_in = [] + + # all edges that end in that processor facility primary vertex, on that day + db_cur2 = main_db_con.cursor() + for row_b in db_cur2.execute("""select edge_id from edges e, vertices v + where e.start_day = {} + and e.d_vertex_id = v.vertex_id + and v.facility_id = {} + and v.storage_vertex = 0 + group by edge_id""".format(day, facility_id)): + input_edge_id = row_b[0] + flow_in.append(flow_var[input_edge_id]) + + logger.debug( + "flow in for capacity constraint on processor facility {} day {}: {}".format(facility_id, day, flow_in)) + prob += lpSum(flow_in) <= daily_inflow_max_capacity * processor_daily_flow_vars[(facility_id, day)], \ + "constraint max flow into processor facility {}, day {}, flow var {}".format( + facility_id, day, processor_daily_flow_vars[facility_id, day]) + + prob += lpSum(flow_in) >= daily_inflow_min_capacity * processor_daily_flow_vars[ + (facility_id, day)], "constraint min flow into processor {}, day {}".format(facility_id, day) + # else: + # pdb.set_trace() + + if is_candidate == 1: + # forces processor build var to be correct + # if there is flow through a candidate processor then it has to be built + prob += processor_build_vars[facility_id] >= processor_daily_flow_vars[ + (facility_id, day)], "constraint forces processor build var to be correct {}, {}".format( + facility_id, processor_build_vars[facility_id]) + + logger.debug("FINISHED: create_constraint_daily_processor_capacity") + return prob + + +# =============================================================================== + + +def create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_primary_processor_vertex_constraints - conservation of flow") + # for all of these vertices, flow in always == flow out + # node_counter = 0 + # node_constraint_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + # total flow in == total flow out, subject to conversion; + # dividing by "required quantity" functionally converts all commodities to the same "processor-specific units" + + # processor primary vertices with input commodity and quantity needed to produce specified output quantities + # 2 sets of constraints; one for the primary processor vertex to cover total flow in and out + # one for each input and output commodity (sum over sources) to ensure its ratio matches facility_commodities + + # the current construction of this method is dependent on having only one input commodity type per processor + # this limitation makes sharing max transport distance from the input to an output commodity feasible + + logger.debug("conservation of flow and commodity ratios, primary processor vertices:") + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' + when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day + when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + ifnull(f.candidate, 0) candidate_check, + e.source_facility_id, + v.source_facility_id, + v.commodity_id, + c.share_max_transport_distance + from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f + join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) + where ft.facility_type = 'processor' + and v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 0 + and v.facility_id = fc.facility_id + and fc.commodity_id = c.commodity_id + and fc.commodity_id = e.commodity_id + group by v.vertex_id, + in_or_out_edge, + constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + candidate_check, + e.source_facility_id, + v.commodity_id, + v.source_facility_id, + ifnull(c.share_max_transport_distance, 'N') + order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id + ;""" + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + sql_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info( + "execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + sql_data = sql_data.fetchall() + logger.info( + "fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + # Nested dictionaries + # flow_in_lists[primary_processor_vertex_id] = dict of commodities handled by that processor vertex + + # flow_in_lists[primary_processor_vertex_id][commodity1] = + # list of edge ids that flow that commodity into that vertex + + # flow_in_lists[vertex_id].values() to get all flow_in edges for all commodities, a list of lists + # if edge out commodity inherits transport distance, then source_facility id must match. if not, aggregate + + flow_in_lists = {} + flow_out_lists = {} + inherit_max_transport = {} + # inherit_max_transport[commodity_id] = 'Y' or 'N' + + for row_a in sql_data: + + vertex_id = row_a[0] + in_or_out_edge = row_a[1] + # constraint_day = row_a[2] + commodity_id = row_a[3] + # mode = row_a[4] + edge_id = row_a[5] + # nx_edge_id = row_a[6] + quantity = float(row_a[7]) + # facility_id = row_a[8] + # commodity_name = row_a[9] + # fc_io_commodity = row_a[10] + # activity_level = row_a[11] + # is_candidate = row_a[12] + edge_source_facility_id = row_a[13] + vertex_source_facility_id = row_a[14] + # v_commodity_id = row_a[15] + inherit_max_transport_distance = row_a[16] + if commodity_id not in inherit_max_transport.keys(): + if inherit_max_transport_distance == 'Y': + inherit_max_transport[commodity_id] = 'Y' + else: + inherit_max_transport[commodity_id] = 'N' + + if in_or_out_edge == 'in': + # if the vertex isn't in the main dict yet, add it + # could have multiple source facilities + # could also have more than one input commodity now + flow_in_lists.setdefault(vertex_id, {}) + flow_in_lists[vertex_id].setdefault((commodity_id, quantity, edge_source_facility_id), []).append(flow_var[edge_id]) + # flow_in_lists[vertex_id] is itself a dict keyed on commodity, quantity (ratio) and edge_source_facility; + # value is a list of edge ids into that vertex of that commodity and edge source + + elif in_or_out_edge == 'out': + # for out-lists, could have multiple commodities as well as multiple sources + # some may have a max transport distance, inherited or independent, some may not + flow_out_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it + flow_out_lists[vertex_id].setdefault((commodity_id, quantity, edge_source_facility_id), []).append(flow_var[edge_id]) + + # Because we keyed on commodity, source facility tracking is merged as we pass through the processor vertex + + # 1) for each output commodity, check against an input to ensure correct ratio - only need one input + # 2) for each input commodity, check against an output to ensure correct ratio - only need one output; + # 2a) first sum sub-flows over input commodity + + # 1---------------------------------------------------------------------- + constrained_input_flow_vars = set([]) + # pdb.set_trace() + + for key, value in iteritems(flow_out_lists): + #value is a dictionary with commodity & source as keys + # set up a dictionary that will be filled with input lists to check ratio against + compare_input_dict = {} + compare_input_dict_commod = {} + vertex_id = key + zero_in = False + #value is a dictionary keyed on output commodity, quantity required, edge source + if vertex_id in flow_in_lists: + in_quantity = 0 + in_commodity_id = 0 + in_source_facility_id = -1 + for ikey, ivalue in iteritems(flow_in_lists[vertex_id]): + in_commodity_id = ikey[0] + in_quantity = ikey[1] + in_source = ikey[2] + # list of edges + compare_input_dict[in_source] = ivalue + # to accommodate and track multiple input commodities; does not keep sources separate + # aggregate lists over sources, by commodity + if in_commodity_id not in compare_input_dict_commod.keys(): + compare_input_dict_commod[in_commodity_id] = set([]) + for edge in ivalue: + compare_input_dict_commod[in_commodity_id].add(edge) + else: + zero_in = True + + + # value is a dict - we loop once here for each output commodity and source at the vertex + for key2, value2 in iteritems(value): + out_commodity_id = key2[0] + out_quantity = key2[1] + out_source = key2[2] + # edge_list = value2 + flow_var_list = value2 + # if we need to match source facility, there is only one set of input lists + # otherwise, use all input lists - this aggregates sources + # need to keep commodities separate, units may be different + # known issue - we could have double-counting problems if only some outputs have to inherit max + # transport distance through this facility + match_source = inherit_max_transport[out_commodity_id] + compare_input_list = [] + if match_source == 'Y': + if len(compare_input_dict_commod.keys()) > 1: + error = "Multiple input commodities for processors and shared max transport distance are" \ + " not supported within the same scenario." + logger.error(error) + raise Exception(error) + + if out_source in compare_input_dict.keys(): + compare_input_list = compare_input_dict[out_source] + # if no valid input edges - none for vertex, or if output needs to match source and there are no + # matching source + if zero_in or (match_source == 'Y' and len(compare_input_list) == 0): + prob += lpSum( + flow_var_list) == 0, "processor flow, vertex {} has zero in so zero out of commodity {} " \ + "with source {} if applicable".format( + vertex_id, out_commodity_id, out_source) + else: + if match_source == 'Y': + # ratio constraint for this output commodity relative to total input of each commodity + required_flow_out = lpSum(flow_var_list) / out_quantity + # check against an input dict + prob += required_flow_out == lpSum( + compare_input_list) / in_quantity, "processor flow, vertex {}, source_facility {}," \ + " commodity {} output quantity" \ + " checked against single input commodity quantity".format( + vertex_id, out_source, out_commodity_id, in_commodity_id) + for flow_var in compare_input_list: + constrained_input_flow_vars.add(flow_var) + else: + for k, v in iteritems(compare_input_dict_commod): + # pdb.set_trace() + # as long as the input source doesn't match an output that needs to inherit + compare_input_list = list(v) + in_commodity_id = k + # ratio constraint for this output commodity relative to total input of each commodity + required_flow_out = lpSum(flow_var_list) / out_quantity + # check against an input dict + prob += required_flow_out == lpSum( + compare_input_list) / in_quantity, "processor flow, vertex {}, source_facility {}," \ + " commodity {} output quantity" \ + " checked against commodity {} input quantity".format( + vertex_id, out_source, out_commodity_id, in_commodity_id) + for flow_var in compare_input_list: + constrained_input_flow_vars.add(flow_var) + + for key, value in iteritems(flow_in_lists): + vertex_id = key + for key2, value2 in iteritems(value): + commodity_id = key2[0] + # out_quantity = key2[1] + source = key2[2] + # edge_list = value2 + flow_var_list = value2 + for flow_var in flow_var_list: + if flow_var not in constrained_input_flow_vars: + prob += flow_var == 0, "processor flow, vertex {} has no matching out edges so zero in of " \ + "commodity {} with source {}".format( + vertex_id, commodity_id, source) + + logger.debug("FINISHED: create_primary_processor_conservation_of_flow_constraints") + return prob + + +# =============================================================================== + + +def create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_var, processor_excess_vars): + logger.debug("STARTING: create_constraint_conservation_of_flow") + # node_counter = 0 + node_constraint_counter = 0 + storage_vertex_constraint_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + logger.info("conservation of flow, storage vertices:") + # storage vertices, any facility type + # these have at most one direction of transport edges, so no need to track mode + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' + when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day + when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id, v.facility_id, c.commodity_name, + v.activity_level, + ft.facility_type + + from vertices v, facility_type_id ft, commodities c, facilities f + join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) + and (e.o_vertex_id = v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) + + where v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 1 + and v.commodity_id = c.commodity_id + + group by v.vertex_id, + in_or_out_edge, + constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id,v.facility_id, c.commodity_name, + v.activity_level + + order by v.facility_id, v.vertex_id, e.edge_id + ;""" + + # get the data from sql and see how long it takes. + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + vertexid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + vertexid_data = vertexid_data.fetchall() + logger.info( + "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_in_lists = {} + flow_out_lists = {} + for row_v in vertexid_data: + vertex_id = row_v[0] + in_or_out_edge = row_v[1] + constraint_day = row_v[2] + commodity_id = row_v[3] + edge_id = row_v[4] + # nx_edge_id = row_v[5] + # facility_id = row_v[6] + # commodity_name = row_v[7] + # activity_level = row_v[8] + facility_type = row_v[9] + + if in_or_out_edge == 'in': + flow_in_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( + flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( + flow_var[edge_id]) + + logger.info("adding processor excess variabless to conservation of flow") + + # add any processors to flow_out_lists if not already created + # Addresses bug documented in #382 + for key, value in iteritems(flow_in_lists): + facility_type = key[3] + if facility_type == 'processor' and key not in flow_out_lists.keys(): + flow_out_lists[key] = [] + + for key, value in iteritems(flow_out_lists): + vertex_id = key[0] + # commodity_id = key[1] + # day = key[2] + facility_type = key[3] + if facility_type == 'processor': + flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) + + for key, value in iteritems(flow_out_lists): + + if key in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum( + flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + else: + prob += lpSum(flow_out_lists[key]) == lpSum( + 0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + for key, value in iteritems(flow_in_lists): + + if key not in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum( + 0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + logger.info( + "total conservation of flow constraints created on nodes: {}".format(storage_vertex_constraint_counter)) + + logger.info("conservation of flow, nx_nodes:") + # for each day, get all edges in and out of the node. + # Sort edges by commodity and whether they're going in or out of the node + sql = """select nn.node_id, + (case when e.from_node_id = nn.node_id then 'out' + when e.to_node_id = nn.node_id then 'in' else 'error' end) in_or_out_edge, + (case when e.from_node_id = nn.node_id then start_day + when e.to_node_id = nn.node_id then end_day else 0 end) constraint_day, + e.commodity_id, + ifnull(mode, 'NULL'), + e.edge_id, nx_edge_id, + miles, + (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, + e.source_facility_id, + e.commodity_id + from networkx_nodes nn + join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) + where nn.location_id is null + order by nn.node_id, e.commodity_id, + (case when e.from_node_id = nn.node_id then start_day + when e.to_node_id = nn.node_id then end_day else 0 end), + in_or_out_edge, e.source_facility_id, e.commodity_id + ;""" + + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + nodeid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info( + "execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t " + "".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + nodeid_data = nodeid_data.fetchall() + logger.info( + "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_in_lists = {} + flow_out_lists = {} + + for row_a in nodeid_data: + node_id = row_a[0] + in_or_out_edge = row_a[1] + constraint_day = row_a[2] + # commodity_id = row_a[3] + mode = row_a[4] + edge_id = row_a[5] + # nx_edge_id = row_a[6] + # miles = row_a[7] + intermodal = row_a[8] + source_facility_id = row_a[9] + commodity_id = row_a[10] + + # node_counter = node_counter +1 + # if node is not intermodal, conservation of flow holds per mode; + # if intermodal, then across modes + if intermodal == 'N': + if in_or_out_edge == 'in': + flow_in_lists.setdefault( + (node_id, intermodal, source_facility_id, constraint_day, commodity_id, mode), []).append( + flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault( + (node_id, intermodal, source_facility_id, constraint_day, commodity_id, mode), []).append( + flow_var[edge_id]) + else: + if in_or_out_edge == 'in': + flow_in_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id), + []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id), + []).append(flow_var[edge_id]) + + for key, value in iteritems(flow_out_lists): + node_id = key[0] + # intermodal_flag = key[1] + source_facility_id = key[2] + day = key[3] + commodity_id = key[4] + if len(key) == 6: + node_mode = key[5] + else: + node_mode = 'intermodal' + if key in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[ + key]), "conservation of flow, nx node {}, " \ + "source facility {}, commodity {}, " \ + "day {}, mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + else: + prob += lpSum(flow_out_lists[key]) == lpSum( + 0), "conservation of flow (zero out), nx node {}, source facility {}, commodity {}, day {}," \ + " mode {}".format(node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + for key, value in iteritems(flow_in_lists): + node_id = key[0] + # intermodal_flag = key[1] + source_facility_id = key[2] + day = key[3] + commodity_id = key[4] + if len(key) == 6: + node_mode = key[5] + else: + node_mode = 'intermodal' + + if key not in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum( + 0), "conservation of flow (zero in), nx node {}, source facility {}, commodity {}, day {}," \ + " mode {}".format(node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + logger.info("total conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) + + # Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints + + logger.debug("FINISHED: create_constraint_conservation_of_flow") + + return prob + + +# =============================================================================== + + +def create_constraint_max_route_capacity(logger, the_scenario, prob, flow_var): + logger.info("STARTING: create_constraint_max_route_capacity") + logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) + # min_capacity_level must be a number from 0 to 1, inclusive + # min_capacity_level is only relevant when background flows are turned on + # it sets a floor to how much capacity can be reduced by volume. + # min_capacity_level = .25 means route capacity will never be less than 25% of full capacity, + # even if "volume" would otherwise restrict it further + # min_capacity_level = 0 allows a route to be made unavailable for FTOT flow if base volume is too high + # this currently applies to all modes + logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + # capacity for storage routes + sql = """select + rr.route_id, sr.storage_max, sr.route_name, e.edge_id, e.start_day + from route_reference rr + join storage_routes sr on sr.route_name = rr.route_name + join edges e on rr.route_id = e.route_id + ;""" + # get the data from sql and see how long it takes. + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + storage_edge_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for storage edges:") + logger.info("execute for edges for storage - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + storage_edge_data = storage_edge_data.fetchall() + logger.info("fetchall edges for storage - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in storage_edge_data: + route_id = row_a[0] + aggregate_storage_capac = row_a[1] + storage_route_name = row_a[2] + edge_id = row_a[3] + start_day = row_a[4] + + flow_lists.setdefault((route_id, aggregate_storage_capac, storage_route_name, start_day), []).append( + flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on storage route {} named {} for day {}".format(key[0], + key[2], + key[3]) + + logger.debug("route_capacity constraints created for all storage routes") + + # capacity for transport routes + # Assumption - all flowing material is in kgal, all flow is summed on a single non-pipeline nx edge + sql = """select e.edge_id, e.nx_edge_id, e.max_edge_capacity, e.start_day, e.simple_mode, e.phase_of_matter, + e.capac_minus_volume_zero_floor + from edges e + where e.max_edge_capacity is not null + and e.simple_mode != 'pipeline' + ;""" + # get the data from sql and see how long it takes. + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + route_capac_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for transport edges:") + logger.info("execute for non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + route_capac_data = route_capac_data.fetchall() + logger.info("fetchall non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in route_capac_data: + edge_id = row_a[0] + nx_edge_id = row_a[1] + nx_edge_capacity = row_a[2] + start_day = row_a[3] + simple_mode = row_a[4] + phase_of_matter = row_a[5] + capac_minus_background_flow = max(row_a[6], 0) + min_restricted_capacity = max(capac_minus_background_flow, nx_edge_capacity * the_scenario.minCapacityLevel) + + if simple_mode in the_scenario.backgroundFlowModes: + use_capacity = min_restricted_capacity + else: + use_capacity = nx_edge_capacity + + # flow is in thousand gallons (kgal), for liquid, or metric tons, for solid + # capacity is in truckload, rail car, barge, or pipeline movement per day + # if mode is road and phase is liquid, capacity is in truckloads per day, we want it in kgal + # ftot_supporting_gis tells us that there are 8 kgal per truckload, + # so capacity * 8 gives us correct units or kgal per day + # => use capacity * ftot_supporting_gis multiplier to get capacity in correct flow units + + multiplier = 1 # if units match, otherwise specified here + if simple_mode == 'road': + if phase_of_matter == 'liquid': + multiplier = the_scenario.truck_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.truck_load_solid.magnitude + elif simple_mode == 'water': + if phase_of_matter == 'liquid': + multiplier = the_scenario.barge_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.barge_load_solid.magnitude + elif simple_mode == 'rail': + if phase_of_matter == 'liquid': + multiplier = the_scenario.railcar_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.railcar_load_solid.magnitude + + converted_capacity = use_capacity * multiplier + + flow_lists.setdefault((nx_edge_id, converted_capacity, start_day), []).append(flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on nx edge {} for day {}".format(key[0], key[2]) + + logger.debug("route_capacity constraints created for all non-pipeline transport routes") + + logger.debug("FINISHED: create_constraint_max_route_capacity") + return prob + + +# =============================================================================== + + +def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_pipeline_capacity") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) + logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + # capacity for pipeline tariff routes + # with sasc, may have multiple flows per segment, slightly diff commodities + sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow allowed_flow, + l.source, e.mode, instr(e.mode, l.source) + from edges e, pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where e.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(e.mode, l.source)>0 + group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source + ;""" + # capacity needs to be shared over link_id for any edge_id associated with that link + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + pipeline_capac_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for transport edges:") + logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + pipeline_capac_data = pipeline_capac_data.fetchall() + logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in pipeline_capac_data: + edge_id = row_a[0] + # tariff_id = row_a[1] + link_id = row_a[2] + # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall + # Link capacity * 42 is now in kgal per day, to match flow in kgal + link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[3] + start_day = row_a[4] + capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[5], 0) + min_restricted_capacity = max(capac_minus_background_flow_kgal, + link_capacity_kgal_per_day * the_scenario.minCapacityLevel) + + # capacity_nodes_mode_source = row_a[6] + edge_mode = row_a[7] + # mode_match_check = row_a[8] + if 'pipeline' in the_scenario.backgroundFlowModes: + link_use_capacity = min_restricted_capacity + else: + link_use_capacity = link_capacity_kgal_per_day + + # add flow from all relevant edges, for one start; may be multiple tariffs + flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format( + key[0], key[3], key[2]) + + logger.debug("pipeline capacity constraints created for all transport routes") + + logger.debug("FINISHED: create_constraint_pipeline_capacity") + return prob + + +# =============================================================================== + + +def setup_pulp_problem(the_scenario, logger): + logger.info("START: setup PuLP problem") + + # flow_var is the flow on each edge by commodity and day. + # the optimal value of flow_var will be solved by PuLP + flow_vars = create_flow_vars(the_scenario, logger) + + # unmet_demand_var is the unmet demand at each destination, being determined + unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) + + # processor_build_vars is the binary variable indicating whether a candidate processor is used + # and thus whether its build cost is charged + processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) + + # binary tracker variables + processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario, logger) + + # tracking unused production + processor_excess_vars = create_processor_excess_output_vars(the_scenario, logger) + + # THIS IS THE OBJECTIVE FUNCTION FOR THE OPTIMIZATION + # ================================================== + + prob = create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) + + prob = create_constraint_unmet_demand(logger, the_scenario, prob, flow_vars, unmet_demand_vars) + + prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) + + # This constraint is being excluded because 1) it is not used in current scenarios and 2) it is not supported by + # this version - it conflicts with the change permitting multiple inputs + # adding back 12/2020 + prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, + processor_vertex_flow_vars) + + prob = create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_vars) + + prob = create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_vars, processor_excess_vars) + + if the_scenario.capacityOn: + prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) + + prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) + + del unmet_demand_vars + + del flow_vars + + # The problem data is written to an .lp file + prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_c2.lp")) + + logger.info("FINISHED: setup PuLP problem") + return prob + + +# =============================================================================== + + +def solve_pulp_problem(prob_final, the_scenario, logger): + import datetime + + logger.info("START: solve_pulp_problem") + start_time = datetime.datetime.now() + from os import dup, dup2, close + f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') + orig_std_out = dup(1) + dup2(f.fileno(), 1) + + # status = prob_final.solve (PULP_CBC_CMD(maxSeconds = i_max_sec, fracGap = d_opt_gap, msg=1)) + # CBC time limit and relative optimality gap tolerance + status = prob_final.solve(PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance + logger.info('Completion code: %d; Solution status: %s; Best obj value found: %s' % ( + status, LpStatus[prob_final.status], value(prob_final.objective))) + + dup2(orig_std_out, 1) + close(orig_std_out) + f.close() + # The problem is solved using PuLP's choice of Solver + + logger.info("completed calling prob.solve()") + logger.info( + "FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + # THIS IS THE SOLUTION + + # The status of the solution is printed to the screen + ##LpStatus key string value numerical value + ##LpStatusOptimal ?Optimal? 1 + ##LpStatusNotSolved ?Not Solved? 0 + ##LpStatusInfeasible ?Infeasible? -1 + ##LpStatusUnbounded ?Unbounded? -2 + ##LpStatusUndefined ?Undefined? -3 + logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) + + logger.result( + "Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format( + float(value(prob_final.objective)))) + + return prob_final + + +# =============================================================================== + +def save_pulp_solution(the_scenario, prob, logger, zero_threshold=0.00001): + import datetime + start_time = datetime.datetime.now() + logger.info("START: save_pulp_solution") + non_zero_variable_count = 0 + + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_cur = db_con.cursor() + # drop the optimal_solution table + # ----------------------------- + db_cur.executescript("drop table if exists optimal_solution;") + + # create the optimal_solution table + # ----------------------------- + db_cur.executescript(""" + create table optimal_solution + ( + variable_name string, + variable_value real + ); + """) + + # insert the optimal data into the DB + # ------------------------------------- + for v in prob.variables(): + if v.varValue is None: + logger.debug("Variable value is none: " + str(v.name)) + else: + if v.varValue > zero_threshold: # eliminates values too close to zero + sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format( + v.name, float(v.varValue)) + db_con.execute(sql) + non_zero_variable_count = non_zero_variable_count + 1 + + # query the optimal_solution table in the DB for each variable we care about + # ---------------------------------------------------------------------------- + sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_processors_count = data.fetchone()[0] + logger.info("number of optimal_processors: {}".format(optimal_processors_count)) + + sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_count = data.fetchone()[0] + logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) + sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_sum = data.fetchone()[0] + logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) + logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) + logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format( + optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) + + + sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" + data = db_con.execute(sql) + optimal_edges_count = data.fetchone()[0] + logger.info("number of optimal edges: {}".format(optimal_edges_count)) + + logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format( + float(value(prob.objective)) - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) + logger.info( + "Total Scenario Cost = (transportation + unmet demand penalty + " + "processor construction): \t ${0:,.0f}".format( + float(value(prob.objective)))) + + logger.info( + "FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + +# =============================================================================== + +def record_pulp_solution(the_scenario, logger): + logger.info("START: record_pulp_solution") + non_zero_variable_count = 0 + + with sqlite3.connect(the_scenario.main_db) as db_con: + + logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) + sql = """ + create table optimal_variables as + select + 'UnmetDemand' as variable_type, + cast(substr(variable_name, 13) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + null as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as original_facility, + null as final_facility, + null as prior_edge, + null as miles_travelled + from optimal_solution + where variable_name like 'UnmetDemand%' + union + select + 'Edge' as variable_type, + cast(substr(variable_name, 6) as int) var_id, + variable_value, + edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, + edges.volume*edges.units_conversion_multiplier as converted_volume, + edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, + edges.edge_type, + commodities.commodity_name, + ov.facility_name as o_facility, + dv.facility_name as d_facility, + o_vertex_id, + d_vertex_id, + from_node_id, + to_node_id, + start_day time_period, + edges.commodity_id, + edges.source_facility_id, + s.source_facility_name, + commodities.units, + variable_name, + edges.nx_edge_id, + edges.mode, + edges.mode_oid, + edges.miles, + null as original_facility, + null as final_facility, + null as prior_edge, + edges.miles_travelled as miles_travelled + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join commodities on edges.commodity_id = commodities.commodity_ID + left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id + left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id + left outer join source_commodity_ref as s on edges.source_facility_id = s.source_facility_id + where variable_name like 'Edge%' + union + select + 'BuildProcessor' as variable_type, + cast(substr(variable_name, 16) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + 'placeholder' as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as original_facility, + null as final_facility, + null as prior_edge, + null as miles_travelled + from optimal_solution + where variable_name like 'Build%'; + """ + db_con.execute("drop table if exists optimal_variables;") + db_con.execute(sql) + + logger.info("FINISH: record_pulp_solution") + +# =============================================================================== + + +def parse_optimal_solution_db(the_scenario, logger): + logger.info("starting parse_optimal_solution") + + optimal_processors = [] + optimal_processor_flows = [] + optimal_route_flows = {} + optimal_unmet_demand = {} + optimal_storage_flows = {} + optimal_excess_material = {} + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # do the Storage Edges + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" + data = db_con.execute(sql) + optimal_storage_edges = data.fetchall() + for edge in optimal_storage_edges: + optimal_storage_flows[edge] = optimal_storage_edges[edge] + + # do the Route Edges + sql = """select + variable_name, variable_value, + cast(substr(variable_name, 6) as int) edge_id, + route_id, start_day time_period, edges.commodity_id, + o_vertex_id, d_vertex_id, + v1.facility_id o_facility_id, + v2.facility_id d_facility_id + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join vertices v1 on edges.o_vertex_id = v1.vertex_id + join vertices v2 on edges.d_vertex_id = v2.vertex_id + where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; + """ + data = db_con.execute(sql) + optimal_route_edges = data.fetchall() + for edge in optimal_route_edges: + + variable_name = edge[0] + + variable_value = edge[1] + + edge_id = edge[2] + + route_id = edge[3] + + time_period = edge[4] + + commodity_flowed = edge[5] + + od_pair_name = "{}, {}".format(edge[8], edge[9]) + + # first time route_id is used on a day or commodity + if route_id not in optimal_route_flows: + optimal_route_flows[route_id] = [[od_pair_name, time_period, commodity_flowed, variable_value]] + + else: # subsequent times route is used on different day or for other commodity + optimal_route_flows[route_id].append([od_pair_name, time_period, commodity_flowed, variable_value]) + + # do the processors + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_candidates_processors = data.fetchall() + for proc in optimal_candidates_processors: + optimal_processors.append(proc) + + # do the processor vertex flows + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'ProcessorVertexFlow%';" + data = db_con.execute(sql) + optimal_processor_flows_sql = data.fetchall() + for proc in optimal_processor_flows_sql: + optimal_processor_flows.append(proc) + + # do the UnmetDemand + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmetdemand = data.fetchall() + for ultimate_destination in optimal_unmetdemand: + v_name = ultimate_destination[0] + v_value = ultimate_destination[1] + + search = re.search('\(.*\)', v_name.replace("'", "")) + + if search: + parts = search.group(0).replace("(", "").replace(")", "").split(",_") + + dest_name = parts[0] + commodity_flowed = parts[2] + if not dest_name in optimal_unmet_demand: + optimal_unmet_demand[dest_name] = {} + + if not commodity_flowed in optimal_unmet_demand[dest_name]: + optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) + else: + optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) + + + logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors + logger.info("length of optimal_processor_flows list: {}".format( + len(optimal_processor_flows))) # a list of optimal processor flows + logger.info("length of optimal_route_flows dict: {}".format( + len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values + logger.info("length of optimal_unmet_demand dict: {}".format( + len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values + + return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material + + diff --git a/program/ftot_pulp_candidate_generation.py b/program/ftot_pulp_candidate_generation.py index 69c7f7e..e81b04c 100644 --- a/program/ftot_pulp_candidate_generation.py +++ b/program/ftot_pulp_candidate_generation.py @@ -1,2586 +1,2587 @@ -# --------------------------------------------------------------------------------------------------- -# Name: pulp c2-spiderweb variant -# -# Purpose: PulP optimization - partial source facility as subcommodity variant -# only creates from source edges for commodities with max transport distance -# other commodities get regular edges to reduce problem size -# --------------------------------------------------------------------------------------------------- - -import datetime -import pdb -import re -import sqlite3 -from collections import defaultdict -from six import iteritems - -from pulp import * - -import ftot_supporting -from ftot_supporting import get_total_runtime_string - -# =================== constants============= -storage = 1 -primary = 0 -fixed_schedule_id = 2 -fixed_route_duration = 0 -THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 - -candidate_processing_facilities = [] - -storage_cost_1 = 0.01 -storage_cost_2 = 0.05 -facility_onsite_storage_max = 10000000000 -facility_onsite_storage_min = 0 -fixed_route_max_daily_capacity = 100000000 -fixed_route_min_daily_capacity = 0 -default_max_capacity = 10000000000 -default_min_capacity = 0 - - -# =============================================================================== -def oc1(the_scenario, logger): - # create vertices, then edges for permitted modes, then set volume & capacity on edges - pre_setup_pulp(logger, the_scenario) - - -def oc2(the_scenario, logger): - import ftot_pulp - # create variables, problem to optimize, and constraints - prob = setup_pulp_problem_candidate_generation(the_scenario, logger) - prob = ftot_pulp.solve_pulp_problem(prob, the_scenario, logger) # imported from ftot_pulp as of 11/19/19 - ftot_pulp.save_pulp_solution(the_scenario, prob, logger, zero_threshold=0.0) # imported from ftot pulp as of 12/03/19 - - -def oc3(the_scenario, logger): - record_pulp_candidate_gen_solution(the_scenario, logger) - from ftot_supporting import post_optimization - post_optimization(the_scenario, 'oc3', logger) - # finalize candidate creation and report out - from ftot_processor import processor_candidates - processor_candidates(the_scenario, logger) - - -def check_max_transport_distance_for_OC_step(the_scenario, logger): - # -------------------------------------------------------------------------- - with sqlite3.connect(the_scenario.main_db) as main_db_con: - sql = "SELECT COUNT(*) FROM commodities WHERE max_transport_distance IS NOT NULL;" - db_cur = main_db_con.execute(sql) - count_data = db_cur.fetchone()[0] - print (count_data) - if count_data == 0: - logger.error("running the OC step requires that at least commodity from the RMPs have a max transport " - "distance specified in the input CSV file.") - logger.warning("Please add a max transport distance (in miles) to the RMP csv file.") - logger.warning("NOTE: run time for the optimization step increases with max transport distance.") - sys.exit() - - -# =============================================================================== - - -def source_as_subcommodity_setup(the_scenario, logger): - logger.info("START: source_as_subcommodity_setup") - # create table source_commodity_ref that only has commodities that can flow out of a facility - # no multi commodity entry - # does include entries even if there's no max transport distance, has a flag to indicate that - multi_commodity_name = "multicommodity" - with sqlite3.connect(the_scenario.main_db) as main_db_con: - main_db_con.executescript(""" - - insert or ignore into commodities(commodity_name) values ('{}'); - - drop table if exists source_commodity_ref - ; - - create table source_commodity_ref(id INTEGER PRIMARY KEY, - source_facility_id integer, - source_facility_name text, - source_facility_type_id integer, --lets us differentiate spiderweb from processor - commodity_id integer, - commodity_name text, - units text, - phase_of_matter text, - max_transport_distance numeric, - max_transport_distance_flag text, - CONSTRAINT unique_source_and_name UNIQUE(commodity_id, source_facility_id)) - ; - - insert or ignore into source_commodity_ref ( - source_facility_id, - source_facility_name, - source_facility_type_id, - commodity_id, - commodity_name, - units, - phase_of_matter, - max_transport_distance, - max_transport_distance_flag) - select - f.facility_id, - f.facility_name, - f.facility_type_id, - c.commodity_id, - c.commodity_name, - c.units, - c.phase_of_matter, - (case when c.max_transport_distance is not null then - c.max_transport_distance else Null end) max_transport_distance, - (case when c.max_transport_distance is not null then 'Y' else 'N' end) max_transport_distance_flag - - from commodities c, facilities f, facility_commodities fc - where f.facility_id = fc.facility_id - and f.ignore_facility = 'false' - and fc.commodity_id = c.commodity_id - and fc.io = 'o' - ; - --this will populated processors as potential sources for facilities, but with - --max_transport_distance_flag set to 'N' - - --candidate_process_commodities data setup - -- adding columns to the candidate_process_commodities table - - - drop table if exists candidate_process_commodities_temp; - - create table candidate_process_commodities_temp as - select c.process_id, c.io, c.commodity_name, c.commodity_id, cast(c.quantity as float) quantity, c.units, - c.phase_of_matter, s.input_commodity, cast(s.output_ratio as float) output_ratio - from candidate_process_commodities c - LEFT OUTER JOIN (select o.commodity_id, o.process_id, i.commodity_id as input_commodity, - cast(o.quantity as float) output_quantity, cast(i.quantity as float) input_quantity, - (cast(o.quantity as float)/cast(i.quantity as float)) as output_ratio from candidate_process_commodities o, - candidate_process_commodities i - where o.io = 'o' - and i.io = 'i' - and o.process_id = i.process_id) s - on (c.commodity_id = s.commodity_id and c.process_id = s.process_id and c.io = 'o') - ; - - drop table if exists candidate_process_commodities; - - create table candidate_process_commodities as - select c.*, b.process_id as best_process_id - from candidate_process_commodities_temp c LEFT OUTER JOIN - (select commodity_id, input_commodity, process_id, max(output_ratio) best_output_ratio - from candidate_process_commodities_temp - where io = 'o' - group by commodity_id, input_commodity) b ON - (c.commodity_id = b.commodity_id and c.process_id = b.process_id and c.io = 'o') - ; - - drop table IF EXISTS candidate_process_commodities_temp; - - - """.format(multi_commodity_name, multi_commodity_name) - ) - - return - - -# =============================================================================== - - -def schedule_avg_availabilities(the_scenario, schedule_dict, schedule_length, logger): - avg_availabilities = {} - - for sched_id, sched_array in schedule_dict.items(): - # find average availability over all schedule days - # populate dictionary of one day schedules w/ availability = avg_availability - avg_availability = sum(sched_array)/schedule_length - avg_availabilities[sched_id] = [avg_availability] - - return avg_availabilities - -# =============================================================================== - - -def generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger): - logger.info("START: generate_all_edges_from_source_facilities") - - # plan to generate start and end days based on nx edge time to traverse and schedule - # can still have route_id, but only for storage routes now; nullable - # should only run for commodities with a max commodity constraint - - multi_commodity_name = "multicommodity" - # initializations - all of these get updated if >0 edges exist - edges_requiring_children = 0 - endcap_edges = 0 - edges_resolved = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - transport_edges_created = 0 - nx_edge_count = 0 - source_based_edges_created = 0 - - commodity_mode_data = main_db_con.execute("select * from commodity_mode;") - commodity_mode_data = commodity_mode_data.fetchall() - commodity_mode_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - commodity_phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - commodity_mode_dict[mode, commodity_id] = allowed_yn - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport' - and children_created in ('N', 'Y', 'E');"""): - source_based_edges_created = row_d[0] - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport';"""): - transport_edges_created = row_d[0] - nx_edge_count = row_d[1] - - current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created - from edges - where edge_type = 'transport' - group by children_created - order by children_created asc;""") - current_edge_data = current_edge_data.fetchall() - for row in current_edge_data: - if row[1] == 'N': - edges_requiring_children = row[0] - elif row[1] == 'Y': - edges_resolved = row[0] - elif row[1] == 'E': - endcap_edges = row[0] - logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) - if source_based_edges_created == edges_resolved + endcap_edges: - edges_requiring_children = 0 - if source_based_edges_created == edges_requiring_children + endcap_edges: - edges_resolved = 0 - if source_based_edges_created == edges_requiring_children + edges_resolved: - endcap_edges = 0 - - logger.info( - '{} transport edges created; {} require children'.format(transport_edges_created, edges_requiring_children)) - - counter = 0 - - # set up a table to keep track of endcap nodes - sql = """ - drop table if exists endcap_nodes; - create table if not exists endcap_nodes( - node_id integer NOT NULL, - - location_id integer, - - --mode of edges that it's an endcap for - mode_source text NOT NULL, - - --facility it's an endcap for - source_facility_id integer NOT NULL, - - --commodities it's an endcap for - commodity_id integer NOT NULL, - - --must have a viable process for this commodity to flag as endcap - process_id integer, - - --is this an endcap to allow conversion at destination, that should not be cleaned up? - destination_yn text, - - CONSTRAINT endcap_key PRIMARY KEY (node_id, mode_source, source_facility_id, commodity_id)) - --the combination of these four (excluding location_id) should be unique, - --and all fields except location_id should be filled - ;""" - db_cur.executescript(sql) - - # create transport edges, only between storage vertices and nodes, based on networkx graph - # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link - # iterate through nx edges: if neither node has a location, create 1 edge per viable commodity - # should also be per day, subject to nx edge schedule - # before creating an edge, check: commodity allowed by nx and max transport distance if not null - # will need nodes per day and commodity? or can I just check that with constraints? - # select data for transport edges - # ****1**** Only edges coming from RMP/source storage vertices - # set distance travelled to miles of edges; set indicator for newly created edges to 'N' - # edge_count_from_source = 1 - # ****2**** only nx_edges coming from entries in edges (check connector nodes) - # set distance travelling to miles of new endge plus existing input edge; set indicator for processed edges - # in edges to 'Y' - # only create new edge if distance travelled is less than allowed - # repeat 2 while there are 'N' edges, for transport edges only - # count loops - # this is now per-vertex - rework it to not all be done in loop, but in sql block - # connector and storage edges can be done exactly as before, in fact need to be done first, now in separate - # method - - while_count = 0 - edge_into_facility_counter = 0 - - destination_fac_type = main_db_con.execute("""select facility_type_id from facility_type_id - where facility_type = 'ultimate_destination';""") - destination_fac_type = int(destination_fac_type.fetchone()[0]) - logger.debug("ultimate_Destination type id: {}".format(destination_fac_type)) - - while edges_requiring_children > 0: - - while_count = while_count + 1 - - # --get nx edges that align with the existing "in edges" - data from nx to create new edges - # --for each of those nx_edges, if they connect to more than one "in edge" in this batch, - # only consider connecting to the shortest - # -- if there is a valid nx_edge to build, the crossing node is not an endcap - # if total miles of the new route is over max transport distance, then endcap - # what if one child goes over max transport and another doesn't - # then the node will get flagged as an endcap, and another path may continue off it, allow both for now - # --check that day and commodity are permitted by nx - - potential_edge_data = main_db_con.execute(""" - select - ch.edge_id as ch_nx_edge_id, - ifnull(CAST(chfn.location_id as integer), 'NULL') fn_location_id, - ifnull(CAST(chtn.location_id as integer), 'NULL') tn_location_id, - ch.mode_source, - p.phase_of_matter, - nec.route_cost, - ch.from_node_id, - ch.to_node_id, - nec.dollar_cost, - ch.miles, - ch.capacity, - ch.artificial, - ch.mode_source_oid, - --parent edge into - p.commodity_id, - p.end_day, - --parent's dest. vertex if exists - ifnull(p.d_vertex_id,0) o_vertex, - p.source_facility_id, - p.leadin_miles_travelled, - - (p.edge_count_from_source +1) as new_edge_count, - (p.total_route_cost + nec.route_cost) new_total_route_cost, - p.edge_id leadin_edge, - p.nx_edge_id leadin_nx_edge, - - --new destination vertex if exists - ifnull(chtv.vertex_id,0) d_vertex, - - c.max_transport_distance, - ifnull(proc.best_process_id, 0), - --type of new destination vertex if exists - ifnull(chtv.facility_type_id,0) d_vertex_type - - from - (select count(edge_id) parents, - min(miles_travelled) leadin_miles_travelled, - * - from edges - where children_created = 'N' - -----------------do not mess with this "group by" - group by to_node_id, source_facility_id, commodity_id, end_day - ------------------it affects which columns we're checking over for min miles travelled - --------------so that we only get the parent edges we want - order by parents desc - ) p, --parent edges to use in this batch - networkx_edges ch, - networkx_edge_costs nec, - commodities c, - networkx_nodes chfn, - networkx_nodes chtn - left outer join vertices chtv - on (CAST(chtn.location_id as integer) = chtv.location_id - and (p.source_facility_id = chtv.source_facility_id or chtv.source_facility_id = 0) - and (chtv.commodity_id = p.commodity_id or chtv.facility_type_id = {}) - and p.end_day = chtv.schedule_day - and chtv.iob = 'i' - and chtv.storage_vertex = 1) - left outer join candidate_process_commodities proc on - (c.commodity_id = proc.input_commodity and best_process_id is not null) - - - where p.to_node_id = ch.from_node_id - --and p.mode = ch.mode_source --build across modes, control at conservation of flow - and ch.to_node_id = chtn.node_id - and ch.from_node_id = chfn.node_id - and p.phase_of_matter = nec.phase_of_matter_id - and ch.edge_id = nec.edge_id - and ifnull(ch.capacity, 1) > 0 - and p.commodity_id = c.commodity_id - ;""".format(destination_fac_type)) - # --should only get a single leadin edge per networkx/source/commodity/day combination - # leadin edge should be in route_data, current set of min. identifiers - # if i'm trying to add an edge that has an entry in route_data, new miles travelled must be less - - potential_edge_data = potential_edge_data.fetchall() - - main_db_con.execute("update edges set children_created = 'Y' where children_created = 'N';") - - - # this deliberately includes updating "parent" edges that did not get chosen because they weren't the - # current shortest path - # those edges are still "resolved" by this batch - - for row_a in potential_edge_data: - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - mode_oid = row_a[12] - commodity_id = row_a[13] - origin_day = row_a[14] - vertex_id = row_a[15] - source_facility_id = row_a[16] - leadin_edge_miles_travelled = row_a[17] - new_edge_count = row_a[18] - total_route_cost = row_a[19] - leadin_edge_id = row_a[20] - to_vertex = row_a[22] - max_commodity_travel_distance = row_a[23] - input_commodity_process_id = row_a[24] - to_vertex_type = row_a[25] - - # end_day = origin_day + fixed_route_duration - new_miles_travelled = miles + leadin_edge_miles_travelled - if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys()\ - and commodity_mode_dict[mode, commodity_id] == 'Y': - if to_vertex_type ==2: - logger.debug('edge {} goes in to location {} at ' - 'node {} with vertex {}'.format(leadin_edge_id, to_location, to_node, to_vertex)) - - if ((new_miles_travelled > max_commodity_travel_distance and input_commodity_process_id != 0) - or to_vertex_type == destination_fac_type): - #False ): - # designate leadin edge as endcap - children_created = 'E' - destination_yn = 'M' - if to_vertex_type == destination_fac_type: - destination_yn = 'Y' - else: - destination_yn = 'N' - # update the incoming edge to indicate it's an endcap - db_cur.execute( - "update edges set children_created = '{}' where edge_id = {}".format(children_created, - leadin_edge_id)) - if from_location != 'NULL': - db_cur.execute("""insert or ignore into endcap_nodes( - node_id, location_id, mode_source, source_facility_id, commodity_id, process_id, destination_yn) - VALUES ({}, {}, '{}', {}, {},{},'{}'); - """.format(from_node, from_location, mode, source_facility_id, commodity_id, - input_commodity_process_id, destination_yn)) - else: - db_cur.execute("""insert or ignore into endcap_nodes( - node_id, mode_source, source_facility_id, commodity_id, process_id, destination_yn) - VALUES ({}, '{}', {}, {},{},'{}'); - """.format(from_node, mode, source_facility_id, commodity_id, input_commodity_process_id, destination_yn)) - - - # designate leadin edge as endcap - # this does, deliberately, allow endcap status to be overwritten if we've found a shorter - # path to a previous endcap - - # create new edge - elif new_miles_travelled <= max_commodity_travel_distance: - - simple_mode = row_a[3].partition('_')[0] - tariff_id = 0 - if simple_mode == 'pipeline': - - # find tariff_ids - - sql = "select mapping_id from pipeline_mapping where id = {} and id_field_name = " \ - "'source_OID' and source = '{}' and mapping_id is not null;".format(mode_oid, mode) - for tariff_row in db_cur.execute(sql): - tariff_id = tariff_row[0] - - # if there are no edges yet for this day, nx, subc combination, - # AND this is the shortest existing leadin option for this day, nx, subc combination - # we'd be creating an edge for (otherwise wait for the shortest option) - # at this step, some leadin edge should always exist - - if origin_day in range(1, schedule_length + 1): - if (origin_day + fixed_route_duration <= schedule_length): # if link is - # traversable in the timeframe - if simple_mode != 'pipeline' or tariff_id >= 0: - # for allowed commodities - - if from_location == 'NULL' and to_location == 'NULL': - # for each day and commodity, get the corresponding origin vertex id to - # include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough - # than checking if facility type is 'ultimate destination' - # new for bsc, only connect to vertices with matching source_Facility_id - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, - total_route_cost) VALUES ({}, {}, - {}, {}, {}, - {}, {}, {}, - '{}',{},'{}',{}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - # only create edge going into a location if an appropriate vertex exists - elif from_location == 'NULL' and to_location != 'NULL' and to_vertex > 0: - edge_into_facility_counter = edge_into_facility_counter + 1 - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, - total_route_cost) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}', {}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - to_vertex, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - elif from_location != 'NULL' and to_location == 'NULL': - # for each day and commodity, get the corresponding origin vertex id to - # include with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough - # than checking if facility type is 'ultimate destination' - # new for bsc, only connect to vertices with matching source facility id ( - # only limited for RMP vertices) - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, - total_route_cost) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - elif from_location != 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding origin and destination - # vertex ids to include with the edge info - - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, - source_facility_id, - miles_travelled, children_created, edge_count_from_source, - total_route_cost) VALUES ({}, {}, - {}, {}, {}, - {}, {}, - {}, {}, {}, - '{}',{},'{}', {}, - {},'{}',{},'{}', - {}, - {},'{}',{},{}); - """.format(from_node, to_node, - origin_day, origin_day + fixed_route_duration, commodity_id, - vertex_id, to_vertex, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, - miles, simple_mode, tariff_id, phase_of_matter, - source_facility_id, - new_miles_travelled, 'N', new_edge_count, total_route_cost)) - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport' - and children_created in ('N', 'Y', 'E');"""): - source_based_edges_created = row_d[0] - - for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) - from edges e where e.edge_type = 'transport';"""): - transport_edges_created = row_d[0] - nx_edge_count = row_d[1] - - current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created - from edges - where edge_type = 'transport' - group by children_created - order by children_created asc;""") - current_edge_data = current_edge_data.fetchall() - for row in current_edge_data: - if row[1] == 'N': - edges_requiring_children = row[0] - elif row[1] == 'Y': - edges_resolved = row[0] - elif row[1] == 'E': - endcap_edges = row[0] - logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) - if source_based_edges_created == edges_resolved + endcap_edges: - edges_requiring_children = 0 - if source_based_edges_created == edges_requiring_children + endcap_edges: - edges_resolved = 0 - if source_based_edges_created == edges_requiring_children + edges_resolved: - endcap_edges = 0 - - if counter % 10 == 0 or edges_requiring_children == 0: - logger.info( - '{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( - transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) - counter = counter + 1 - - logger.info('{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( - transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) - logger.info("all source-based transport edges created") - - logger.info("create an index for the edges table") - for row in db_cur.execute("select count(*) from endcap_nodes"): - logger.debug("{} endcap nodes before cleanup".format(row)) - - sql = (""" - CREATE INDEX IF NOT EXISTS edge_index ON edges ( - edge_id, route_id, from_node_id, to_node_id, commodity_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id); - """) - db_cur.execute(sql) - return - - -# =============================================================================== - - -def clean_up_endcaps(the_scenario, logger): - logger.info("START: clean_up_endcaps") - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - logger.info("clean up endcap node flagging") - - sql = (""" - drop table if exists e2; - - create table e2 as - select e.edge_id , - e.route_id , - e.from_node_id , - e.to_node_id , - e.start_day , - e.end_day , - e.commodity_id , - e.o_vertex_id , - e.d_vertex_id , - e.max_edge_capacity , - e.volume , - e.capac_minus_volume_zero_floor , - e.min_edge_capacity , - e.capacity_units , - e.units_conversion_multiplier , - e.edge_flow_cost , - e.edge_flow_cost2 , - e.edge_type , - e.nx_edge_id , - e.mode , - e.mode_oid , - e.miles , - e.simple_mode , - e.tariff_id , - e.phase_of_matter , - e.source_facility_id , - e.miles_travelled , - (case when w.cleaned_children = 'C' then w.cleaned_children else e.children_created end) as children_created, - edge_count_from_source , - total_route_cost - from edges e - left outer join - (select 'C' as cleaned_children, t.edge_id - from edges, - (select e.edge_id - from edges e, edges e2, endcap_nodes en - where e.children_created = 'E' - and e.to_node_id = e2.from_node_id - and e.source_facility_id = e2.source_facility_id - and e.commodity_id = e2.commodity_id - and e.miles_travelled < e2.miles_travelled - and e.to_node_id = en.node_id - and e.source_facility_id = en.source_facility_id - and e.commodity_id = en.commodity_id - and en.destination_yn = 'N' - group by e.edge_id - order by e.edge_id - ) t - where edges.edge_id = t.edge_id - and edges.children_created = 'E' - ) w on e.edge_id = w.edge_id - ; - - delete from edges; - - insert into edges select * from e2; - - drop table e2; - """) - logger.info("calling sql on update end caps in the edges table") - db_cur.executescript(sql) - - logger.info("clean up the endcap_nodes table") - - sql = (""" - drop table if exists en_clean; - - create table en_clean - as select en.* - from endcap_nodes en, edges e - where e.to_node_id = en.node_id - and e.commodity_id = en.commodity_id - and e.source_facility_id = en.source_facility_id - and e.children_created = 'E' - ; - - drop table endcap_nodes; - create table endcap_nodes as select * from en_clean; - - drop table en_clean; - - """) - logger.info("calling sql to clean up the endcap_nodes table") - db_cur.executescript(sql) - - for row in db_cur.execute("select count(*) from endcap_nodes;"): - logger.debug("{} endcap nodes after cleanup".format(row)) - - return - - -# =============================================================================== - - -def generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger): - logger.info("START: generate_all_edges_without_max_commodity_constraint") - # make sure this covers edges from an RMP if the commodity has no max transport distance - - multi_commodity_name = "multicommodity" - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - db_cur = main_db_con.cursor() - - commodity_mode_data = main_db_con.execute("select * from commodity_mode;") - commodity_mode_data = commodity_mode_data.fetchall() - commodity_mode_dict = {} - for row in commodity_mode_data: - mode = row[0] - commodity_id = int(row[1]) - commodity_phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - commodity_mode_dict[mode, commodity_id] = allowed_yn - - counter = 0 - - # for all commodities with no max transport distance or output by a candidate process - source_facility_id = 0 - - # create transport edges, only between storage vertices and nodes, based on networkx graph - # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link - # iterate through nx edges: create 1 edge per viable commodity as allowed by nx phase_of_matter or allowed - # commodities. Should also be per day, subject to nx edge schedule - # if there is a max transport distance on a candidate process output, disregard for now because we don't know - # where the candidates will be located - - # select data for transport edges - - sql = """select - ne.edge_id, - ifnull(fn.location_id, 'NULL'), - ifnull(tn.location_id, 'NULL'), - ne.mode_source, - ifnull(nec.phase_of_matter_id, 'NULL'), - nec.route_cost, - ne.from_node_id, - ne.to_node_id, - nec.dollar_cost, - ne.miles, - ne.capacity, - ne.artificial, - ne.mode_source_oid - from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec - - where ne.from_node_id = fn.node_id - and ne.to_node_id = tn.node_id - and ne.edge_id = nec.edge_id - and ifnull(ne.capacity, 1) > 0 - ;""" - nx_edge_data = main_db_con.execute(sql) - nx_edge_data = nx_edge_data.fetchall() - for row_a in nx_edge_data: - - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - mode_oid = row_a[12] - simple_mode = row_a[3].partition('_')[0] - - counter = counter + 1 - - tariff_id = 0 - if simple_mode == 'pipeline': - - # find tariff_ids - sql = "select mapping_id from pipeline_mapping where id = {} and id_field_name = 'source_OID' and " \ - "source = '{}' and mapping_id is not null;".format(mode_oid, mode) - for tariff_row in db_cur.execute(sql): - tariff_id = tariff_row[0] - - if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys()\ - and commodity_mode_dict[mode, commodity_id] == 'Y': - - # Edges are placeholders for flow variables - # if both ends have no location, iterate through viable commodities and days, create edge - # for all days (restrict by link schedule if called for) - # for all allowed commodities, as currently defined by link phase of matter - for day in range(1, schedule_length + 1): - if (day + fixed_route_duration <= schedule_length): # if link is traversable in the - # timeframe - if simple_mode != 'pipeline' or tariff_id >= 0: - # for allowed commodities that can be output by some facility or process in the scenario - for row_c in db_cur.execute("""select commodity_id - from source_commodity_ref - where phase_of_matter = '{}' - and max_transport_distance_flag = 'N' - group by commodity_id - union - select commodity_id - from candidate_process_commodities - where phase_of_matter = '{}' - and io = 'o' - group by commodity_id""".format(phase_of_matter, phase_of_matter)): - db_cur4 = main_db_con.cursor() - commodity_id = row_c[0] - # source_facility_id = row_c[1] # fixed to 0 for all edges created by this method - - if from_location == 'NULL' and to_location == 'NULL': - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - - elif from_location != 'NULL' and to_location == 'NULL': - # for each day and commodity, get the corresponding origin vertex id to include - # with the edge info - # origin vertex must not be "ultimate_destination - # transport link outgoing from facility - checking fc.io is more thorough than - # checking if facility type is 'ultimate destination' - # new for bsc, only connect to vertices with matching source_facility_id (only - # limited for RMP vertices) - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and - v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): - from_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - from_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - elif from_location == 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding destination vertex id to - # include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and - v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): - to_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - to_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - elif from_location != 'NULL' and to_location != 'NULL': - # for each day and commodity, get the corresponding origin and destination vertex - # ids to include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and - v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): - from_vertex_id = row_d[0] - db_cur5 = main_db_con.cursor() - for row_e in db_cur5.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and - v.source_facility_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): - to_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, - to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, - phase_of_matter, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, {}, - {}, {}, {}, - '{}',{},'{}', {},{},'{}',{},'{}',{}); - """.format(from_node, to_node, - day, day + fixed_route_duration, commodity_id, - from_vertex_id, to_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, - tariff_id, phase_of_matter, source_facility_id)) - - logger.debug("all transport edges created") - - logger.info("all edges created") - logger.info("create an index for the edges table by nodes") - index_start_time = datetime.datetime.now() - sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( - edge_id, route_id, from_node_id, to_node_id, commodity_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") - db_cur.execute(sql) - logger.info("edge_index Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(index_start_time))) - return - - -# =============================================================================== - - -def pre_setup_pulp(logger, the_scenario): - logger.info("START: pre_setup_pulp for candidate generation step") - from ftot_pulp import commodity_mode_setup - - commodity_mode_setup(the_scenario, logger) - - check_max_transport_distance_for_OC_step(the_scenario, logger) - - source_as_subcommodity_setup(the_scenario, logger) - - from ftot_pulp import generate_schedules - logger.debug("----- Using generate_all_vertices method imported from ftot_pulp ------") - schedule_dict, schedule_length = generate_schedules(the_scenario, logger) - - # Re-create schedule dictionary with one-day schedules with availability = average availability - schedule_avg = schedule_avg_availabilities(the_scenario, schedule_dict, schedule_length, logger) - schedule_avg_length = 1 - - from ftot_pulp import generate_all_vertices - logger.debug("----- Using generate_all_vertices method imported from ftot_pulp ------") - generate_all_vertices(the_scenario, schedule_avg, schedule_avg_length, logger) - - from ftot_pulp import add_storage_routes - logger.debug("----- Using add_storage_routes method imported from ftot_pulp ------") - add_storage_routes(the_scenario, logger) - - from ftot_pulp import generate_connector_and_storage_edges - logger.debug("----- Using generate_connector_and_storage_edges method imported from ftot_pulp ------") - generate_connector_and_storage_edges(the_scenario, logger) - - from ftot_pulp import generate_first_edges_from_source_facilities - logger.debug("----- Using generate_first_edges_from_source_facilities method imported from ftot_pulp ------") - generate_first_edges_from_source_facilities(the_scenario, schedule_avg_length, logger) - - generate_all_edges_from_source_facilities(the_scenario, schedule_avg_length, logger) - - clean_up_endcaps(the_scenario, logger) - - generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_avg_length, logger) - - logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) - - from ftot_pulp import set_edges_volume_capacity - logger.debug("----- Using set_edges_volume_capacity method imported from ftot_pulp ------") - set_edges_volume_capacity(the_scenario, logger) - - return - - -# =============================================================================== - - -def create_flow_vars(the_scenario, logger): - logger.info("START: create_flow_vars") - - # call helper method to get list of unique IDs from the Edges table. - # use a the rowid as a simple unique integer index - edge_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, source_facility_id - from edges;""") - edge_list_data = edge_list_cur.fetchall() - counter = 0 - for row in edge_list_data: - if counter % 500000 == 0: - logger.info( - "processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) - counter += 1 - # create an edge for each commodity allowed on this link - this construction may change as specific - # commodity restrictions are added - edge_list.append((row[0])) - - # flow_var is the flow on each arc, being determined; this can be defined any time after all_arcs is defined - flow_var = LpVariable.dicts("Edge", edge_list, 0, None) - logger.detailed_debug("DEBUG: Size of flow_var: {:,.0f}".format(sys.getsizeof(flow_var))) - - return flow_var - - -# =============================================================================== - - -def create_unmet_demand_vars(the_scenario, logger): - logger.info("START: create_unmet_demand_vars") - demand_var_list = [] - # may create vertices with zero demand, but only for commodities that the facility has demand for at some point - #checks material incoming from storage vertex - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) - top_level_commodity_name, v.udp - from vertices v, commodities c, facility_type_id ft, facilities f - where v.commodity_id = c.commodity_id - and ft.facility_type = "ultimate_destination" - and v.storage_vertex = 0 - and v.facility_type_id = ft.facility_type_id - and v.facility_id = f.facility_id - and f.ignore_facility = 'false' - group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) - ;""".format('')): - # facility_id, day, and simplified commodity name, udp - demand_var_list.append((row[0], row[1], row[2], row[3])) - - unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) - - return unmet_demand_var - - -# =============================================================================== - - -def create_candidate_processor_build_vars(the_scenario, logger): - logger.info("START: create_candidate_processor_build_vars") - processors_build_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute( - "select f.facility_id from facilities f, facility_type_id ft where f.facility_type_id = " - "ft.facility_type_id and facility_type = 'processor' and candidate = 1 and ignore_facility = 'false' " - "group by facility_id;"): - # grab all candidate processor facility IDs - processors_build_list.append(row[0]) - - processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list, 0, None, 'Binary') - - return processor_build_var - - -# =============================================================================== - - -def create_binary_processor_vertex_flow_vars(the_scenario, logger): - logger.info("START: create_binary_processor_vertex_flow_vars") - processors_flow_var_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 0 - group by v.facility_id, v.schedule_day;"""): - # facility_id, day - processors_flow_var_list.append((row[0], row[1])) - - processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0, None, 'Binary') - - return processor_flow_var - - -# =============================================================================== - - -def create_processor_excess_output_vars(the_scenario, logger): - logger.info("START: create_processor_excess_output_vars") - excess_var_list = [] - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - xs_cur = db_cur.execute(""" - select vertex_id, commodity_id - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 1;""") - # facility_id, day, and simplified commodity name - xs_data = xs_cur.fetchall() - for row in xs_data: - excess_var_list.append(row[0]) - - excess_var = LpVariable.dicts("XS", excess_var_list, 0, None) - - return excess_var - - -# =============================================================================== - - -def create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars): - logger.debug("START: create_opt_problem") - prob = LpProblem("Flow assignment", LpMinimize) - - logger.detailed_debug("DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_vars))) - logger.detailed_debug("DEBUG: length of flow_vars: {}".format(len(flow_vars))) - logger.detailed_debug("DEBUG: length of processor_build_vars: {}".format(len(processor_build_vars))) - - unmet_demand_costs = [] - flow_costs = {} - logger.detailed_debug("DEBUG: start loop through sql to append unmet_demand_costs") - for u in unmet_demand_vars: - udp = u[3] - unmet_demand_costs.append(udp * unmet_demand_vars[u]) - logger.detailed_debug("DEBUG: finished loop through sql to append unmet_demand_costs. total records: {}".format( - len(unmet_demand_costs))) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - logger.detailed_debug("DEBUG: start sql execute to get flow cost data") - # Flow cost memory improvements: only get needed data; dict instead of list; narrow in lpsum - flow_cost_var = db_cur.execute("select edge_id, edge_flow_cost from edges e group by edge_id;") - logger.detailed_debug("DEBUG: start the fetchall") - flow_cost_data = flow_cost_var.fetchall() - logger.detailed_debug( - "DEBUG: start iterating through {:,.0f} flow_cost_data records".format(len(flow_cost_data))) - counter = 0 - for row in flow_cost_data: - edge_id = row[0] - edge_flow_cost = row[1] - counter += 1 - - # flow costs cover transportation and storage - flow_costs[edge_id] = edge_flow_cost - logger.detailed_debug( - "DEBUG: finished loop through sql to append flow costs: total records: {:,.0f}".format(len(flow_costs))) - - logger.detailed_debug("debug: start prob+= unmet_demand_costs + flow cost + no processor build costs in" - "candidate generation step") - prob += (lpSum(unmet_demand_costs) + lpSum( - flow_costs[k] * flow_vars[k] for k in flow_costs)), "Total Cost of Transport, storage, and penalties" - logger.detailed_debug("debug: done prob+= unmet_demand_costs + flow cost + no processor build costs in " - "candidate generation step") - - logger.debug("FINISHED: create_opt_problem") - return prob - - -# =============================================================================== - - -def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): - logger.debug("START: create_constraint_unmet_demand") - - # apply activity_level to get corresponding actual demand for var - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - # var has form(facility_name, day, simple_fuel) - # unmet demand commodity should be simple_fuel = supertype - logger.detailed_debug("DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_var))) - - demand_met_dict = defaultdict(list) - actual_demand_dict = {} - - # want to specify that all edges leading into this vertex + unmet demand = total demand - # demand primary (non-storage) vertices - - db_cur = main_db_con.cursor() - # each row_a is a primary vertex whose edges in contributes to the met demand of var - # will have one row for each fuel subtype in the scenario - unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - from vertices v, commodities c, facility_type_id ft, facilities f, edges e - where v.facility_id = f.facility_id - and ft.facility_type = 'ultimate_destination' - and f.facility_type_id = ft.facility_type_id - and f.ignore_facility = 'false' - and v.facility_type_id = ft.facility_type_id - and v.storage_vertex = 0 - and c.commodity_id = v.commodity_id - and e.d_vertex_id = v.vertex_id - group by v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - ;""") - - unmet_data = unmet_data.fetchall() - for row_a in unmet_data: - var_full_demand = row_a[2] - proportion_of_supertype = row_a[3] - var_activity_level = row_a[4] - facility_id = row_a[6] - day = row_a[7] - top_level_commodity = row_a[8] - udp = row_a[9] - edge_id = row_a[10] - var_actual_demand = var_full_demand * var_activity_level - - # next get inbound edges and apply appropriate modifier proportion to get how much of var's demand they - # satisfy - demand_met_dict[(facility_id, day, top_level_commodity, udp)].append( - flow_var[edge_id] * proportion_of_supertype) - actual_demand_dict[(facility_id, day, top_level_commodity, udp)] = var_actual_demand - - for key in unmet_demand_var: - if key in demand_met_dict: - # then there are some edges in - prob += lpSum(demand_met_dict[key]) == actual_demand_dict[key] - unmet_demand_var[ - key], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(key[0], - key[1], - key[2]) - else: - if key not in actual_demand_dict: - pdb.set_trace() - # no edges in, so unmet demand equals full demand - prob += actual_demand_dict[key] == unmet_demand_var[ - key], "constraint set unmet demand variable for facility {}, day {}, commodity {} - " \ - "no edges able to meet demand".format( - key[0], key[1], key[2]) - - logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") - return prob - - -# =============================================================================== - - -def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - # force flow out of origins to be <= supply - - # for each primary (non-storage) supply vertex - # flow out of a vertex <= supply of the vertex, true for every day and commodity - # Assumption - each RMP produces a single commodity - # Assumption - only one vertex exists per day per RMP (no multi commodity or subcommodity) - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row_a in db_cur.execute("""select vertex_id, activity_level, supply - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and ft.facility_type = 'raw_material_producer' - and storage_vertex = 0;"""): - supply_vertex_id = row_a[0] - activity_level = row_a[1] - max_daily_supply = row_a[2] - actual_vertex_supply = activity_level * max_daily_supply - - flow_out = [] - db_cur2 = main_db_con.cursor() - # select all edges leaving that vertex and sum their flows - # should be a single connector edge - for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): - edge_id = row_b[0] - flow_out.append(flow_var[edge_id]) - - prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format( - actual_vertex_supply, supply_vertex_id) - - logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") - return prob - - -# =============================================================================== - - -def create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_var): - logger.info("STARTING: create_primary_processor_vertex_constraints - capacity and conservation of flow") - # for all of these vertices, flow in always == flow out - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - # total flow in == total flow out, subject to conversion; - # summed over commodities but not days; does not check commodity proportions - # dividing by "required quantity" functionally converts all commodities to the same "processor-specific units" - - # processor primary vertices with input commodity and quantity needed to produce specified output quantities - # 2 sets of constraints; one for the primary processor vertex to cover total flow in and out - # one for each input and output commodity (sum over sources) to ensure its ratio matches facility_commodities - - logger.debug("conservation of flow and commodity ratios, primary processor vertices:") - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' - end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 - end) constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - ifnull(f.candidate, 0) candidate_check, - e.source_facility_id, - v.source_facility_id, - v.commodity_id - from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f - join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) - where ft.facility_type = 'processor' - and v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 0 - and v.facility_id = fc.facility_id - and fc.commodity_id = c.commodity_id - and fc.commodity_id = e.commodity_id - group by v.vertex_id, - in_or_out_edge, - constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - candidate_check, - e.source_facility_id, - v.commodity_id, - v.source_facility_id - order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id - ;""" - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - sql_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info( - "execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t " - "".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - sql_data = sql_data.fetchall() - logger.info( - "fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - # Nested dictionaries - # flow_in_lists[primary_processor_vertex_id] = dict of commodities handled by that processor vertex - - # flow_in_lists[primary_processor_vertex_id][commodity1] = - # list of edge ids that flow that commodity into that vertex - - # flow_in_lists[vertex_id].values() to get all flow_in edges for all commodities, a list of lists - - flow_in_lists = {} - flow_out_lists = {} - - for row_a in sql_data: - - vertex_id = row_a[0] - in_or_out_edge = row_a[1] - commodity_id = row_a[3] - edge_id = row_a[5] - quantity = float(row_a[7]) - - if in_or_out_edge == 'in': - flow_in_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it - flow_in_lists[vertex_id].setdefault((commodity_id, quantity), []).append(flow_var[edge_id]) - # flow_in_lists[vertex_id] is itself a dict keyed on commodity and quantity; - # value is a list of edge ids into that vertex of that commodity - - elif in_or_out_edge == 'out': - flow_out_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it - flow_out_lists[vertex_id].setdefault((commodity_id, quantity), []).append(flow_var[edge_id]) - - # Because we keyed on commodity, source facility tracking is merged as we pass through the processor vertex - - # 1) for each output commodity, check against an input to ensure correct ratio - only need one input - # 2) for each input commodity, check against an output to ensure correct ratio - only need one output; - # 2a) first sum sub-flows over input commodity - # 3)-x- handled by daily processor capacity constraint [calculate total processor input to ensure - # compliance with capacity constraint - - # 1---------------------------------------------------------------------- - - for key, value in iteritems(flow_out_lists): - vertex_id = key - zero_in = False - if vertex_id in flow_in_lists: - compare_input_list = [] - in_quantity = 0 - in_commodity_id = 0 - for ikey, ivalue in iteritems(flow_in_lists[vertex_id]): - in_commodity_id = ikey[0] - in_quantity = ikey[1] - # edge_list = value2 - compare_input_list = ivalue - else: - zero_in = True - - # value is a dict - we loop once here for each output commodity at the vertex - for key2, value2 in iteritems(value): - out_commodity_id = key2[0] - out_quantity = key2[1] - # edge_list = value2 - flow_var_list = value2 - if zero_in: - prob += lpSum( - flow_var_list) == 0, "processor flow, vertex {} has zero in so zero out of commodity {}".format( - vertex_id, out_commodity_id) - else: - # ratio constraint for this output commodity relative to total input - required_flow_out = lpSum(flow_var_list) / out_quantity - # check against an input dict - prob += required_flow_out == lpSum( - compare_input_list) / in_quantity, "processor flow, vertex {}, commodity {} output quantity " \ - "checked against commodity {} input quantity".format( - vertex_id, out_commodity_id, in_commodity_id) - - # 2---------------------------------------------------------------------- - for key, value in iteritems(flow_in_lists): - vertex_id = key - zero_out = False - if vertex_id in flow_out_lists: - compare_output_list = [] - out_quantity = 0 - for okey, ovalue in iteritems(flow_out_lists[vertex_id]): - out_commodity_id = okey[0] - out_quantity = okey[1] - # edge_list = value2 - compare_output_list = ovalue - else: - zero_out = True - - # value is a dict - we loop once here for each input commodity at the vertex - for key2, value2 in iteritems(value): - in_commodity_id = key2[0] - in_quantity = key2[1] - # edge_list = value2 - flow_var_list = value2 - if zero_out: - prob += lpSum( - flow_var_list) == 0, "processor flow, vertex {} has zero out so zero in of commodity {}".format( - vertex_id, in_commodity_id) - else: - # ratio constraint for this output commodity relative to total input - required_flow_in = lpSum(flow_var_list) / in_quantity - # check against an input dict - prob += required_flow_in == lpSum( - compare_output_list) / out_quantity, "processor flow, vertex {}, commodity {} input quantity " \ - "checked against commodity {} output quantity".format( - vertex_id, in_commodity_id, out_commodity_id) - - logger.debug("FINISHED: create_primary_processor_conservation_of_flow_constraints") - return prob - - -# =============================================================================== - - -def create_constraint_conservation_of_flow_storage_vertices(logger, the_scenario, prob, flow_var, - processor_excess_vars): - logger.debug("STARTING: create_constraint_conservation_of_flow_storage_vertices") - storage_vertex_constraint_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - logger.info("conservation of flow, storage vertices:") - # storage vertices, any facility type - # these have at most one direction of transport edges, so no need to track mode - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' - end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 - end) constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id, v.facility_id, c.commodity_name, - v.activity_level, - ft.facility_type - - from vertices v, facility_type_id ft, commodities c, facilities f - join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) and (e.o_vertex_id = - v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) - - where v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 1 - and v.commodity_id = c.commodity_id - - group by v.vertex_id, - in_or_out_edge, - constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id,v.facility_id, c.commodity_name, - v.activity_level - - order by v.facility_id, v.vertex_id, e.edge_id - ;""" - - # get the data from sql and see how long it takes. - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - vertexid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - vertexid_data = vertexid_data.fetchall() - logger.info("fetchall storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_in_lists = {} - flow_out_lists = {} - for row_v in vertexid_data: - vertex_id = row_v[0] - in_or_out_edge = row_v[1] - constraint_day = row_v[2] - commodity_id = row_v[3] - edge_id = row_v[4] - facility_type = row_v[9] - - if in_or_out_edge == 'in': - flow_in_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( - flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( - flow_var[edge_id]) - - logger.info("adding processor excess variables to conservation of flow") - for key, value in iteritems(flow_out_lists): - vertex_id = key[0] - facility_type = key[3] - if facility_type == 'processor': - flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) - - for key, value in iteritems(flow_out_lists): - - if key in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum( - flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - else: - prob += lpSum(flow_out_lists[key]) == lpSum( - 0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - for key, value in iteritems(flow_in_lists): - - if key not in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum( - 0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], - key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - logger.info("total conservation of flow constraints created on storage vertices: {}".format( - storage_vertex_constraint_counter)) - - return prob - - -# =============================================================================== - -def create_constraint_conservation_of_flow_endcap_nodes(logger, the_scenario, prob, flow_var, - processor_excess_vars): - # This creates constraints for all non-vertex nodes, with variant rules for endcaps nodes - logger.debug("STARTING: create_constraint_conservation_of_flow_endcap_nodes") - node_constraint_counter = 0 - passthrough_constraint_counter = 0 - other_endcap_constraint_counter = 0 - endcap_no_reverse_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - logger.info("conservation of flow, nx_nodes, with endcaps:") - # non-vertex nodes, no facility type, connect 2 transport edges over matching days and commodities - # for each day, get all edges in and out of the node. Sort them by commodity and whether they're going in or - # out of the node - - # retrieve candiate process information - # process_dict[input commodity id] = list [proc_id, quantity, tuples (output comm. id, quantity)] - # initialze Value as list containing process id, then add outputs - process_dict = {} - process_outputs_dict = {} - - # list each output with its input to key off; don't use process id - # nested dictionary: first key is input commodity id - # second key is output commodity id; value is list with preferred process: - # # process_id,input quant, output quantity, output ratio, with the largest output ratio for that commodity pair - # list each output with its input to key on - sql = """select process_id, commodity_id, quantity - from candidate_process_commodities - where io = 'i';""" - process_inputs = db_cur.execute(sql) - process_inputs = process_inputs.fetchall() - - for row in process_inputs: - process_id = row[0] - commodity_id = row[1] - quantity = row[2] - process_dict.setdefault(commodity_id, [process_id, quantity]) - process_dict[commodity_id] = [process_id, quantity] - - - sql = """select o.process_id, o.commodity_id, o.quantity, i.commodity_id - from candidate_process_commodities o, candidate_process_commodities i - where o.io = 'o' - and i.process_id = o.process_id - and i.io = 'i';""" - process_outputs = db_cur.execute(sql) - process_outputs = process_outputs.fetchall() - - for row in process_outputs: - process_id = row[0] - output_commodity_id = row[1] - quantity = row[2] - input_commodity_id = row[3] - - process_dict[input_commodity_id].append((output_commodity_id, quantity)) - process_outputs_dict.setdefault(process_id, []).append(output_commodity_id) - - sql = """select nn.node_id, - (case when e.from_node_id = nn.node_id then 'out' when e.to_node_id = nn.node_id then 'in' else 'error' - end) in_or_out_edge, - (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 - end) constraint_day, - e.commodity_id, - ifnull(mode, 'NULL'), - e.edge_id, nx_edge_id, - miles, - (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, - e.source_facility_id, - e.commodity_id, - (case when en.node_id is null then 'N' else 'E' end) as endcap, - (case when en.node_id is null then 0 else en.process_id end) as endcap_process, - (case when en.node_id is null then 0 else en.source_facility_id end) as endcap_source_facility - from networkx_nodes nn - join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) - left outer join endcap_nodes en on (nn.node_id = en.node_id and e.commodity_id = en.commodity_id and - e.source_facility_id = en.source_facility_id) - where nn.location_id is null - order by nn.node_id, e.commodity_id, - (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 - end), - in_or_out_edge, e.source_facility_id, e.commodity_id - ;""" - - # get the data from sql and see how long it takes. - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - nodeid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info( - "execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t " - "".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - nodeid_data = nodeid_data.fetchall() - logger.info( - "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_in_lists = {} - flow_out_lists = {} - endcap_ref = {} - - for row_a in nodeid_data: - node_id = row_a[0] - in_or_out_edge = row_a[1] - constraint_day = row_a[2] - commodity_id = row_a[3] - mode = row_a[4] - edge_id = row_a[5] - nx_edge_id = row_a[6] - miles = row_a[7] - intermodal = row_a[8] - source_facility_id = row_a[9] - commodity_id = row_a[10] - endcap_flag = row_a[11] # is 'E' if endcap, else 'Y' or 'null' - endcap_input_process = row_a[12] - endcap_source_facility = row_a[13] - # endcap ref is keyed on node_id, makes a list of [commodity_id, list, list]; - # first list will be source facilities - if endcap_source_facility > 0: - endcap_ref.setdefault(node_id, [endcap_input_process, commodity_id, [], []]) - if endcap_source_facility not in endcap_ref[node_id][2]: - endcap_ref[node_id][2].append(endcap_source_facility) - - # if node is not intermodal, conservation of flow holds per mode; - # if intermodal, then across modes - # if node is endcap, conservation of flow holds across source facilities - if intermodal == 'N': - if in_or_out_edge == 'in': - flow_in_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id, - endcap_input_process, mode), []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id, - endcap_input_process, mode), []).append(flow_var[edge_id]) - else: - if in_or_out_edge == 'in': - flow_in_lists.setdefault( - (node_id, intermodal, source_facility_id, constraint_day, commodity_id, endcap_input_process), - []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault( - (node_id, intermodal, source_facility_id, constraint_day, commodity_id, endcap_input_process), - []).append(flow_var[edge_id]) - - endcap_dict = {} - - for node, value in iteritems(endcap_ref): - # add output commodities to endcap_ref[node][3] - # endcap_ref[node][2] is a list of source facilities this endcap matches for the input commodity - process_id = value[0] - if process_id > 0: - endcap_ref[node][3] = process_outputs_dict[process_id] - - # if this node has at least one edge flowing out - for key, value in iteritems(flow_in_lists): - - if node_constraint_counter < 50000: - if node_constraint_counter % 5000 == 0: - logger.info("flow constraint created for {} nodes".format(node_constraint_counter)) - # logger.info('{} edges constrained'.format(edge_counter) - - if node_constraint_counter % 50000 == 0: - if node_constraint_counter == 50000: - logger.info("5000 nodes constrained, switching to log every 50000 instead of 5000") - logger.info("flow constraint created for {} nodes".format(node_constraint_counter)) - # logger.info('{} edges constrained'.format(edge_counter)) - - node_id = key[0] - intermodal_flag = key[1] - source_facility_id = key[2] - day = key[3] - commodity_id = key[4] - endcap_input_process = key[5] - - if len(key) == 7: - node_mode = key[6] - else: - node_mode = 'intermodal' - - # if this node is an endcap - # if this commodity and source match the inputs for the endcap process - # or if this output commodity matches an input process for this node, enter the first loop - # if output commodity, don't do anything - will be handled via input commodity - if (node_id in endcap_ref and - ((endcap_input_process == endcap_ref[node_id][0] and commodity_id == endcap_ref[node_id][ - 1] and source_facility_id in endcap_ref[node_id][2]) - or commodity_id in endcap_ref[node_id][3])): - # if we need to handle this commodity & source according to the endcap process - # and it is an input or output of the current process - # if this is an output of a different process, it fails the above "if" and gets a standard constraint - # below - if commodity_id in process_dict and (node_id, day) not in endcap_dict: - - # tracking which endcap nodes we've already made constraints for - endcap_dict[node_id, day] = commodity_id - - # else, this is an output of this candidate process, and it's endcap constraint is created by the - # input commodity - # which must have edges in, or this would never have been tagged as an endcap node as long as the - # input commodity has some flow in - - # for this node, on this day, with this source facility - intermodal and mode and encapflag - # unchanged, commodity may not - # for endcap nodes, can't check for flow in exactly the same way because commodity may change - input_quantity = process_dict[commodity_id][1] - # since we're working with input to an endcap, need all matching inputs, which may include other - # source facilities as specified in endcap_ref[node_id][2] - in_key_list = [key] - if len(endcap_ref[node_id][2]) > 1: - for alt_source in endcap_ref[node_id][2]: - if alt_source != source_facility_id: - - if intermodal_flag == 'N': - new_key = ( - node_id, intermodal_flag, alt_source, day, commodity_id, endcap_input_process, - node_mode) - else: - new_key = ( - node_id, intermodal_flag, alt_source, day, commodity_id, endcap_input_process) - - in_key_list.append(new_key) - - # this is an endcap for this commodity and source, so unprocessed flow out is not allowed - # unless it's from a different source - - prob += lpSum(flow_out_lists[k] for k in in_key_list if k in flow_out_lists) == lpSum( - 0), "conservation of flow (zero allowed out, must be processed), endcap, nx node {}, " \ - "source facility {}, commodity {}, day {}, mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - - outputs_dict = {} - # starting with the 3rd item in the list, the first output commodity tuple - for i in range(2, len(process_dict[commodity_id])): - output_commodity_id = process_dict[commodity_id][i][0] - output_quantity = process_dict[commodity_id][i][1] - outputs_dict[output_commodity_id] = output_quantity - # now we have a dict of allowed outputs for this input; construct keys for matching flows - # allow cross-modal flow?no, these are not candidates yet, want to use exists routes - - for o_commodity_id, quant in iteritems(outputs_dict): - # creating one constraint per output commodity, per endcap node - node_constraint_counter = node_constraint_counter + 1 - output_source_facility_id = 0 - output_process_id = 0 - - # setting up the outflow edges to check - if node_mode == 'intermodal': - out_key = ( - node_id, intermodal_flag, output_source_facility_id, day, o_commodity_id, - output_process_id) - else: - out_key = ( - node_id, intermodal_flag, output_source_facility_id, day, o_commodity_id, - output_process_id, node_mode) - - if out_key not in flow_out_lists: - # node has edges flowing in, but no valid out edges; restrict flow in to zero - prob += lpSum(flow_in_lists[k] for k in in_key_list if k in flow_in_lists) == lpSum( - 0), "conservation of flow (zero allowed in), endcap, nx node {}, source facility {}, " \ - "commodity {}, day {}, mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - other_endcap_constraint_counter = other_endcap_constraint_counter + 1 - else: - # out_key is in flow_out_lists - # endcap functioning as virtual processor - # now aggregate flows for this commodity - # processsed_flow_out = lpSum(flow_out_lists[out_key])/quant - # divide by /input_quantity for the required processing ratio - - # if key is in flow_out_lists, some input material may pass through unprocessed - # if out_key is in flow_in_lists, some output material may pass through unprocessed - # difference is the amount of input material processed: lpSum(flow_in_lists[key]) - - # lpSum(flow_out_lists[key]) - agg_inflow_lists = [] - for k in in_key_list: - if k in flow_in_lists: - for l in flow_in_lists[k]: - if l not in agg_inflow_lists: - agg_inflow_lists.append(l) - - if out_key in flow_in_lists: - prob += lpSum(agg_inflow_lists) / input_quantity == ( - lpSum(flow_out_lists[out_key]) - lpSum(flow_in_lists[out_key])) / quant, \ - "conservation of flow, endcap as processor, passthrough processed,nx node {}," \ - "source facility {}, in commodity {}, out commodity {}, day {},mode {}".format( - node_id, source_facility_id, commodity_id, o_commodity_id, day, node_mode) - passthrough_constraint_counter = passthrough_constraint_counter + 1 - else: - prob += lpSum(agg_inflow_lists) / input_quantity == lpSum(flow_out_lists[ - out_key]) / quant, \ - "conservation of flow, endcap as processor, no passthrough, nx node {}, " \ - "source facility {}, in commodity {}, out commodity {}, day {}, " \ - "mode {}".format( - node_id, source_facility_id, commodity_id, o_commodity_id, day, - node_mode) - other_endcap_constraint_counter = other_endcap_constraint_counter + 1 - - - - - - else: - # if this node is not an endcap, or has no viable candidate process, handle as standard - # if this node has at least one edge flowing in, and out, ensure flow is compliant - # here is where we need to add the processor ratios and allow conversion, for endcaps - if key in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum(flow_out_lists[ - key]), "conservation of flow, nx node {}, " \ - "source facility {}, commodity {}, day {}, " \ - "mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - # node has edges flowing in, but not out; restrict flow in to zero - else: - prob += lpSum(flow_in_lists[key]) == lpSum( - 0), "conservation of flow (zero allowed in), nx node {}, source facility {}, commodity {}, " \ - "day {}, mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - for key, value in iteritems(flow_out_lists): - node_id = key[0] - source_facility_id = key[2] - day = key[3] - commodity_id = key[4] - if len(key) == 7: - node_mode = key[6] - else: - node_mode = 'intermodal' - - # node has edges flowing out, but not in; restrict flow out to zero - if key not in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum( - 0), "conservation of flow (zero allowed out), nx node {}, source facility {}, commodity {}, " \ - "day {}, mode {}".format( - node_id, source_facility_id, commodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - logger.info( - "total non-endcap conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) - - # Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints - - logger.debug("endcap constraints: passthrough {}, other {}, total no reverse {}".format( - passthrough_constraint_counter, other_endcap_constraint_counter, endcap_no_reverse_counter)) - logger.info("FINISHED: create_constraint_conservation_of_flow_endcap_nodes") - - return prob - - -# =============================================================================== - - -def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_pipeline_capacity") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) - logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - # capacity for pipeline tariff routes - # with source tracking, may have multiple flows per segment, slightly diff commodities - sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow - allowed_flow, l.source, e.mode, instr(e.mode, l.source) - from edges e, pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where e.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(e.mode, l.source)>0 - group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source - ;""" - # capacity needs to be shared over link_id for any edge_id associated with that link - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - pipeline_capac_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for transport edges:") - logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - pipeline_capac_data = pipeline_capac_data.fetchall() - logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( - get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in pipeline_capac_data: - edge_id = row_a[0] - tariff_id = row_a[1] - link_id = row_a[2] - # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall - # Link capacity * 42 is now in kgal per day, to match flow in kgal - link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[3] - start_day = row_a[4] - capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[5], 0) - min_restricted_capacity = max(capac_minus_background_flow_kgal, - link_capacity_kgal_per_day * the_scenario.minCapacityLevel) - - capacity_nodes_mode_source = row_a[6] - edge_mode = row_a[7] - mode_match_check = row_a[8] - if 'pipeline' in the_scenario.backgroundFlowModes: - link_use_capacity = min_restricted_capacity - else: - link_use_capacity = link_capacity_kgal_per_day - - # add flow from all relevant edges, for one start; may be multiple tariffs - flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format( - key[0], key[3], key[2]) - - logger.debug("pipeline capacity constraints created for all transport routes") - - logger.debug("FINISHED: create_constraint_pipeline_capacity") - return prob - - -# =============================================================================== - - -def setup_pulp_problem_candidate_generation(the_scenario, logger): - logger.info("START: setup PuLP problem") - - # flow_var is the flow on each edge by commodity and day. - # the optimal value of flow_var will be solved by PuLP - flow_vars = create_flow_vars(the_scenario, logger) - - # unmet_demand_var is the unmet demand at each destination, being determined - unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) - - # processor_build_vars is the binary variable indicating whether a candidate processor is used and thus its build - # cost charged - processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) - - # binary tracker variables for whether a processor is used - # if used, it must abide by capacity constraints, and include build cost if it is a candidate - processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario, logger) - - # tracking unused production - processor_excess_vars = create_processor_excess_output_vars(the_scenario, logger) - - # THIS IS THE OBJECTIVE FUNCTION FOR THE OPTIMIZATION - # ================================================== - - prob = create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) - logger.detailed_debug("DEBUG: size of prob: {}".format(sys.getsizeof(prob))) - - prob = create_constraint_unmet_demand(logger, the_scenario, prob, flow_vars, unmet_demand_vars) - logger.detailed_debug("DEBUG: size of prob: {}".format(sys.getsizeof(prob))) - - prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) - - from ftot_pulp import create_constraint_daily_processor_capacity - logger.debug("----- Using create_constraint_daily_processor_capacity method imported from ftot_pulp ------") - prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, - processor_vertex_flow_vars) - - prob = create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_vars) - - prob = create_constraint_conservation_of_flow_storage_vertices(logger, the_scenario, prob, - flow_vars, processor_excess_vars) - - prob = create_constraint_conservation_of_flow_endcap_nodes(logger, the_scenario, prob, flow_vars, - processor_excess_vars) - - if the_scenario.capacityOn: - logger.info("calling create_constraint_max_route_capacity") - logger.debug('using create_constraint_max_route_capacity method from ftot_pulp') - from ftot_pulp import create_constraint_max_route_capacity - prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) - - logger.info("calling create_constraint_pipeline_capacity") - prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) - - del unmet_demand_vars - - del flow_vars - - # SCENARIO SPECIFIC CONSTRAINTS - - # The problem data is written to an .lp file - prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_c2.lp")) - logger.info("FINISHED: setup PuLP problem for candidate generation") - - return prob - - -# =============================================================================== - - -def solve_pulp_problem(prob_final, the_scenario, logger): - import datetime - - logger.info("START: solve_pulp_problem") - start_time = datetime.datetime.now() - from os import dup, dup2, close - f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') - orig_std_out = dup(1) - dup2(f.fileno(), 1) - - # and relative optimality gap tolerance - status = prob_final.solve(PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance - print('Completion code: %d; Solution status: %s; Best obj value found: %s' % ( - status, LpStatus[prob_final.status], value(prob_final.objective))) - - dup2(orig_std_out, 1) - close(orig_std_out) - f.close() - # The problem is solved using PuLP's choice of Solver - - logger.info("completed calling prob.solve()") - logger.info( - "FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - # THIS IS THE SOLUTION - - # The status of the solution is printed to the screen - # LpStatus key string value numerical value - # LpStatusOptimal ?Optimal? 1 - # LpStatusNotSolved ?Not Solved? 0 - # LpStatusInfeasible ?Infeasible? -1 - # LpStatusUnbounded ?Unbounded? -2 - # LpStatusUndefined ?Undefined? -3 - logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) - - logger.result( - "Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format( - float(value(prob_final.objective)))) - - return prob_final - - -# =============================================================================== - - -def save_pulp_solution(the_scenario, prob, logger): - import datetime - - logger.info("START: save_pulp_solution") - non_zero_variable_count = 0 - - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_cur = db_con.cursor() - # drop the optimal_solution table - # ----------------------------- - db_cur.executescript("drop table if exists optimal_solution;") - - # create the optimal_solution table - # ----------------------------- - db_cur.executescript(""" - create table optimal_solution - ( - variable_name string, - variable_value real - ); - """) - - # insert the optimal data into the DB - # ------------------------------------- - for v in prob.variables(): - if v.varValue > 0.0: - sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format( - v.name, float(v.varValue)) - db_con.execute(sql) - non_zero_variable_count = non_zero_variable_count + 1 - - # query the optimal_solution table in the DB for each variable we care about - # ---------------------------------------------------------------------------- - sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_processors_count = data.fetchone()[0] - logger.info("number of optimal_processors: {}".format(optimal_processors_count)) - - sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_count = data.fetchone()[0] - logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) - sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_sum = data.fetchone()[0] - logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) - logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) - logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format( - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) - - - sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" - data = db_con.execute(sql) - optimal_edges_count = data.fetchone()[0] - logger.info("number of optimal edges: {}".format(optimal_edges_count)) - logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format( - float(value(prob.objective)) - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) - logger.info( - "Total Scenario Cost = (transportation + unmet demand penalty + " - "processor construction): \t ${0:,.0f}".format( - float(value(prob.objective)))) - start_time = datetime.datetime.now() - logger.info( - "FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - -# =============================================================================== - - -def record_pulp_candidate_gen_solution(the_scenario, logger): - logger.info("START: record_pulp_candidate_gen_solution") - non_zero_variable_count = 0 - - with sqlite3.connect(the_scenario.main_db) as db_con: - - logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) - sql = """ - create table optimal_variables as - select - 'UnmetDemand' as variable_type, - cast(substr(variable_name, 13) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - null as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as edge_count_from_source, - null as miles_travelled - from optimal_solution - where variable_name like 'UnmetDemand%' - union - select - 'Edge' as variable_type, - cast(substr(variable_name, 6) as int) var_id, - variable_value, - edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, - edges.volume*edges.units_conversion_multiplier as converted_volume, - edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, - edges.edge_type, - commodities.commodity_name, - ov.facility_name as o_facility, - dv.facility_name as d_facility, - o_vertex_id, - d_vertex_id, - from_node_id, - to_node_id, - start_day time_period, - edges.commodity_id, - edges.source_facility_id, - s.source_facility_name, - commodities.units, - variable_name, - edges.nx_edge_id, - edges.mode, - edges.mode_oid, - edges.miles, - edges.edge_count_from_source, - edges.miles_travelled - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join commodities on edges.commodity_id = commodities.commodity_ID - left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id - left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id - left outer join source_commodity_ref as s on edges.source_facility_id = s.source_facility_id - where variable_name like 'Edge%' - union - select - 'BuildProcessor' as variable_type, - cast(substr(variable_name, 16) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - 'placeholder' as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as edge_count_from_source, - null as miles_travelled - from optimal_solution - where variable_name like 'Build%'; - """ - db_con.execute("drop table if exists optimal_variables;") - db_con.executescript(sql) - - - sql = """ - create table optimal_variables_c as - select * from optimal_variables - ; - drop table if exists candidate_nodes; - - create table candidate_nodes as - select * - from - - (select pl.minsize, pl.process_id, ov.to_node_id, ov.mode_oid, ov.mode, ov.commodity_id, - sum(variable_value) agg_value, count(variable_value) num_aggregated - from optimal_variables ov, candidate_process_list pl, candidate_process_commodities pc - - where pc.io = 'i' - and pl.process_id = pc.process_id - and ov.commodity_id = pc.commodity_id - and ov.edge_type = 'transport' - group by ov.mode_oid, ov.to_node_id, ov.commodity_id, pl.process_id, pl.minsize, ov.mode) i, - - (select pl.min_aggregation, pl.process_id, ov.from_node_id, ov.commodity_id, - sum(variable_value) agg_value, count(variable_value) num_aggregated, min(edge_count_from_source) as - edge_count, - o_vertex_id - from optimal_variables ov, candidate_process_list pl, candidate_process_commodities pc - where pc.io = 'i' - and pl.process_id = pc.process_id - and ov.commodity_id = pc.commodity_id - and ov.edge_type = 'transport' - group by ov.from_node_id, ov.commodity_id, pl.process_id, pl.min_aggregation, ov.o_vertex_id) o - left outer join networkx_nodes nn - on o.from_node_id = nn.node_id - - where (i.to_node_id = o.from_node_id - and i.process_id = o.process_id - and i.commodity_id = o.commodity_id - and o.agg_value > i.agg_value - and o.agg_value >= o.min_aggregation) - or (o.o_vertex_id is not null and o.agg_value >= o.min_aggregation) - - group by o.min_aggregation, o.process_id, o.from_node_id, o.commodity_id, o.agg_value, o.num_aggregated, - o.o_vertex_id - order by o.agg_value, edge_count - ; - - - """ - db_con.execute("drop table if exists optimal_variables_c;") - db_con.executescript(sql) - - sql_check_candidate_node_count = "select count(*) from candidate_nodes" - db_cur = db_con.execute(sql_check_candidate_node_count) - candidate_node_count = db_cur.fetchone()[0] - if candidate_node_count > 0: - logger.info("number of candidate nodes identified: {}".format(candidate_node_count)) - else: - logger.warning("the candidate nodes table is empty. Hints: " - "(1) increase the maximum raw material transport distance to allow additional flows to " - "aggregate and meet the minimum candidate facility size" - "(2) increase the ammount of material available to flow from raw material producers") - - logger.info("FINISH: record_pulp_candidate_gen_solution") - - -# =============================================================================== - - -def parse_optimal_solution_db(the_scenario, logger): - logger.info("starting parse_optimal_solution") - - optimal_processors = [] - optimal_processor_flows = [] - optimal_route_flows = {} - optimal_unmet_demand = {} - optimal_storage_flows = {} - optimal_excess_material = {} - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # do the Storage Edges - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" - data = db_con.execute(sql) - optimal_storage_edges = data.fetchall() - for edge in optimal_storage_edges: - optimal_storage_flows[edge] = optimal_storage_edges[edge] - - # do the Route Edges - sql = """select - variable_name, variable_value, - cast(substr(variable_name, 6) as int) edge_id, - route_ID, start_day time_period, edges.commodity_id, - o_vertex_id, d_vertex_id, - v1.facility_id o_facility_id, - v2.facility_id d_facility_id - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join vertices v1 on edges.o_vertex_id = v1.vertex_id - join vertices v2 on edges.d_vertex_id = v2.vertex_id - where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; - """ - data = db_con.execute(sql) - optimal_route_edges = data.fetchall() - for edge in optimal_route_edges: - - variable_value = edge[1] - - route_id = edge[3] - - time_period = edge[4] - - commodity_flowed = edge[5] - - od_pair_name = "{}, {}".format(edge[8], edge[9]) - - if route_id not in optimal_route_flows: # first time route_id is used on a day or commodity - optimal_route_flows[route_id] = [[od_pair_name, time_period, commodity_flowed, variable_value]] - - else: # subsequent times route is used on different day or for other commodity - optimal_route_flows[route_id].append([od_pair_name, time_period, commodity_flowed, variable_value]) - - # do the processors - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_candidates_processors = data.fetchall() - for proc in optimal_candidates_processors: - optimal_processors.append(proc) - - # do the processor vertex flows - sql = "select variable_name, variable_value from optimal_solution where variable_name like " \ - "'ProcessorVertexFlow%';" - data = db_con.execute(sql) - optimal_processor_flows_sql = data.fetchall() - for proc in optimal_processor_flows_sql: - optimal_processor_flows.append(proc) - # optimal_biorefs.append(v.name[22:(v.name.find(",")-1)])# find the name from the - - # do the UnmetDemand - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmetdemand = data.fetchall() - for ultimate_destination in optimal_unmetdemand: - v_name = ultimate_destination[0] - v_value = ultimate_destination[1] - - search = re.search('\(.*\)', v_name.replace("'", "")) - - if search: - parts = search.group(0).replace("(", "").replace(")", "").split(",_") - - dest_name = parts[0] - commodity_flowed = parts[2] - if not dest_name in optimal_unmet_demand: - optimal_unmet_demand[dest_name] = {} - - if not commodity_flowed in optimal_unmet_demand[dest_name]: - optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) - else: - optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) - - - logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors - logger.info("length of optimal_processor_flows list: {}".format( - len(optimal_processor_flows))) # a list of optimal processor flows - logger.info("length of optimal_route_flows dict: {}".format( - len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values - logger.info("length of optimal_unmet_demand dict: {}".format( - len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values - - return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material +# --------------------------------------------------------------------------------------------------- +# Name: pulp c2-spiderweb variant +# +# Purpose: PulP optimization - partial source facility as subcommodity variant +# only creates from source edges for commodities with max transport distance +# other commodities get regular edges to reduce problem size +# --------------------------------------------------------------------------------------------------- + +import datetime +import pdb +import re +import sqlite3 +from collections import defaultdict +from six import iteritems + +from pulp import * + +import ftot_supporting +from ftot_supporting import get_total_runtime_string + +# =================== constants============= +storage = 1 +primary = 0 +fixed_schedule_id = 2 +fixed_route_duration = 0 +THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 + +candidate_processing_facilities = [] + +storage_cost_1 = 0.01 +storage_cost_2 = 0.05 +facility_onsite_storage_max = 10000000000 +facility_onsite_storage_min = 0 +fixed_route_max_daily_capacity = 100000000 +fixed_route_min_daily_capacity = 0 +default_max_capacity = 10000000000 +default_min_capacity = 0 + + +# =============================================================================== +def oc1(the_scenario, logger): + # create vertices, then edges for permitted modes, then set volume & capacity on edges + pre_setup_pulp(logger, the_scenario) + + +def oc2(the_scenario, logger): + import ftot_pulp + # create variables, problem to optimize, and constraints + prob = setup_pulp_problem_candidate_generation(the_scenario, logger) + prob = ftot_pulp.solve_pulp_problem(prob, the_scenario, logger) # imported from ftot_pulp as of 11/19/19 + ftot_pulp.save_pulp_solution(the_scenario, prob, logger, zero_threshold=0.0) # imported from ftot pulp as of 12/03/19 + + +def oc3(the_scenario, logger): + record_pulp_candidate_gen_solution(the_scenario, logger) + from ftot_supporting import post_optimization + post_optimization(the_scenario, 'oc3', logger) + # finalize candidate creation and report out + from ftot_processor import processor_candidates + processor_candidates(the_scenario, logger) + + +def check_max_transport_distance_for_OC_step(the_scenario, logger): + # -------------------------------------------------------------------------- + with sqlite3.connect(the_scenario.main_db) as main_db_con: + sql = "SELECT COUNT(*) FROM commodities WHERE max_transport_distance IS NOT NULL;" + db_cur = main_db_con.execute(sql) + count_data = db_cur.fetchone()[0] + print (count_data) + if count_data == 0: + logger.error("running the OC step requires that at least commodity from the RMPs have a max transport " + "distance specified in the input CSV file.") + logger.warning("Please add a max transport distance (in miles) to the RMP csv file.") + logger.warning("NOTE: run time for the optimization step increases with max transport distance.") + sys.exit() + + +# =============================================================================== + + +def source_as_subcommodity_setup(the_scenario, logger): + logger.info("START: source_as_subcommodity_setup") + # create table source_commodity_ref that only has commodities that can flow out of a facility + # no multi commodity entry + # does include entries even if there's no max transport distance, has a flag to indicate that + multi_commodity_name = "multicommodity" + with sqlite3.connect(the_scenario.main_db) as main_db_con: + main_db_con.executescript(""" + + insert or ignore into commodities(commodity_name) values ('{}'); + + drop table if exists source_commodity_ref + ; + + create table source_commodity_ref(id INTEGER PRIMARY KEY, + source_facility_id integer, + source_facility_name text, + source_facility_type_id integer, --lets us differentiate spiderweb from processor + commodity_id integer, + commodity_name text, + units text, + phase_of_matter text, + max_transport_distance numeric, + max_transport_distance_flag text, + CONSTRAINT unique_source_and_name UNIQUE(commodity_id, source_facility_id)) + ; + + insert or ignore into source_commodity_ref ( + source_facility_id, + source_facility_name, + source_facility_type_id, + commodity_id, + commodity_name, + units, + phase_of_matter, + max_transport_distance, + max_transport_distance_flag) + select + f.facility_id, + f.facility_name, + f.facility_type_id, + c.commodity_id, + c.commodity_name, + c.units, + c.phase_of_matter, + (case when c.max_transport_distance is not null then + c.max_transport_distance else Null end) max_transport_distance, + (case when c.max_transport_distance is not null then 'Y' else 'N' end) max_transport_distance_flag + + from commodities c, facilities f, facility_commodities fc + where f.facility_id = fc.facility_id + and f.ignore_facility = 'false' + and fc.commodity_id = c.commodity_id + and fc.io = 'o' + ; + --this will populated processors as potential sources for facilities, but with + --max_transport_distance_flag set to 'N' + + --candidate_process_commodities data setup + -- TODO as of 2-25 this will throw an error if run repeatedly, + -- adding columns to the candidate_process_commodities table + + + drop table if exists candidate_process_commodities_temp; + + create table candidate_process_commodities_temp as + select c.process_id, c.io, c.commodity_name, c.commodity_id, cast(c.quantity as float) quantity, c.units, + c.phase_of_matter, s.input_commodity, cast(s.output_ratio as float) output_ratio + from candidate_process_commodities c + LEFT OUTER JOIN (select o.commodity_id, o.process_id, i.commodity_id as input_commodity, + cast(o.quantity as float) output_quantity, cast(i.quantity as float) input_quantity, + (cast(o.quantity as float)/cast(i.quantity as float)) as output_ratio from candidate_process_commodities o, + candidate_process_commodities i + where o.io = 'o' + and i.io = 'i' + and o.process_id = i.process_id) s + on (c.commodity_id = s.commodity_id and c.process_id = s.process_id and c.io = 'o') + ; + + drop table if exists candidate_process_commodities; + + create table candidate_process_commodities as + select c.*, b.process_id as best_process_id + from candidate_process_commodities_temp c LEFT OUTER JOIN + (select commodity_id, input_commodity, process_id, max(output_ratio) best_output_ratio + from candidate_process_commodities_temp + where io = 'o' + group by commodity_id, input_commodity) b ON + (c.commodity_id = b.commodity_id and c.process_id = b.process_id and c.io = 'o') + ; + + drop table IF EXISTS candidate_process_commodities_temp; + + + """.format(multi_commodity_name, multi_commodity_name) + ) + + return + + +# =============================================================================== + + +def schedule_avg_availabilities(the_scenario, schedule_dict, schedule_length, logger): + avg_availabilities = {} + + for sched_id, sched_array in schedule_dict.items(): + # find average availability over all schedule days + # populate dictionary of one day schedules w/ availability = avg_availability + avg_availability = sum(sched_array)/schedule_length + avg_availabilities[sched_id] = [avg_availability] + + return avg_availabilities + +# =============================================================================== + + +def generate_all_edges_from_source_facilities(the_scenario, schedule_length, logger): + logger.info("START: generate_all_edges_from_source_facilities") + + # plan to generate start and end days based on nx edge time to traverse and schedule + # can still have route_id, but only for storage routes now; nullable + # should only run for commodities with a max commodity constraint + + multi_commodity_name = "multicommodity" + # initializations - all of these get updated if >0 edges exist + edges_requiring_children = 0 + endcap_edges = 0 + edges_resolved = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + transport_edges_created = 0 + nx_edge_count = 0 + source_based_edges_created = 0 + + commodity_mode_data = main_db_con.execute("select * from commodity_mode;") + commodity_mode_data = commodity_mode_data.fetchall() + commodity_mode_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + commodity_phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + commodity_mode_dict[mode, commodity_id] = allowed_yn + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport' + and children_created in ('N', 'Y', 'E');"""): + source_based_edges_created = row_d[0] + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport';"""): + transport_edges_created = row_d[0] + nx_edge_count = row_d[1] + + current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created + from edges + where edge_type = 'transport' + group by children_created + order by children_created asc;""") + current_edge_data = current_edge_data.fetchall() + for row in current_edge_data: + if row[1] == 'N': + edges_requiring_children = row[0] + elif row[1] == 'Y': + edges_resolved = row[0] + elif row[1] == 'E': + endcap_edges = row[0] + logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) + if source_based_edges_created == edges_resolved + endcap_edges: + edges_requiring_children = 0 + if source_based_edges_created == edges_requiring_children + endcap_edges: + edges_resolved = 0 + if source_based_edges_created == edges_requiring_children + edges_resolved: + endcap_edges = 0 + + logger.info( + '{} transport edges created; {} require children'.format(transport_edges_created, edges_requiring_children)) + + counter = 0 + + # set up a table to keep track of endcap nodes + sql = """ + drop table if exists endcap_nodes; + create table if not exists endcap_nodes( + node_id integer NOT NULL, + + location_id integer, + + --mode of edges that it's an endcap for + mode_source text NOT NULL, + + --facility it's an endcap for + source_facility_id integer NOT NULL, + + --commodities it's an endcap for + commodity_id integer NOT NULL, + + --must have a viable process for this commodity to flag as endcap + process_id integer, + + --is this an endcap to allow conversion at destination, that should not be cleaned up? + destination_yn text, + + CONSTRAINT endcap_key PRIMARY KEY (node_id, mode_source, source_facility_id, commodity_id)) + --the combination of these four (excluding location_id) should be unique, + --and all fields except location_id should be filled + ;""" + db_cur.executescript(sql) + + # create transport edges, only between storage vertices and nodes, based on networkx graph + # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link + # iterate through nx edges: if neither node has a location, create 1 edge per viable commodity + # should also be per day, subject to nx edge schedule + # before creating an edge, check: commodity allowed by nx and max transport distance if not null + # will need nodes per day and commodity? or can I just check that with constraints? + # select data for transport edges + # ****1**** Only edges coming from RMP/source storage vertices + # set distance travelled to miles of edges; set indicator for newly created edges to 'N' + # edge_count_from_source = 1 + # ****2**** only nx_edges coming from entries in edges (check connector nodes) + # set distance travelling to miles of new endge plus existing input edge; set indicator for processed edges + # in edges to 'Y' + # only create new edge if distance travelled is less than allowed + # repeat 2 while there are 'N' edges, for transport edges only + # count loops + # this is now per-vertex - rework it to not all be done in loop, but in sql block + # connector and storage edges can be done exactly as before, in fact need to be done first, now in separate + # method + + while_count = 0 + edge_into_facility_counter = 0 + + destination_fac_type = main_db_con.execute("""select facility_type_id from facility_type_id + where facility_type = 'ultimate_destination';""") + destination_fac_type = int(destination_fac_type.fetchone()[0]) + logger.debug("ultimate_Destination type id: {}".format(destination_fac_type)) + + while edges_requiring_children > 0: + + while_count = while_count + 1 + + # --get nx edges that align with the existing "in edges" - data from nx to create new edges + # --for each of those nx_edges, if they connect to more than one "in edge" in this batch, + # only consider connecting to the shortest + # -- if there is a valid nx_edge to build, the crossing node is not an endcap + # if total miles of the new route is over max transport distance, then endcap + # what if one child goes over max transport and another doesn't + # then the node will get flagged as an endcap, and another path may continue off it, allow both for now + # --check that day and commodity are permitted by nx + + potential_edge_data = main_db_con.execute(""" + select + ch.edge_id as ch_nx_edge_id, + ifnull(CAST(chfn.location_id as integer), 'NULL') fn_location_id, + ifnull(CAST(chtn.location_id as integer), 'NULL') tn_location_id, + ch.mode_source, + p.phase_of_matter, + nec.route_cost, + ch.from_node_id, + ch.to_node_id, + nec.dollar_cost, + ch.miles, + ch.capacity, + ch.artificial, + ch.mode_source_oid, + --parent edge into + p.commodity_id, + p.end_day, + --parent's dest. vertex if exists + ifnull(p.d_vertex_id,0) o_vertex, + p.source_facility_id, + p.leadin_miles_travelled, + + (p.edge_count_from_source +1) as new_edge_count, + (p.total_route_cost + nec.route_cost) new_total_route_cost, + p.edge_id leadin_edge, + p.nx_edge_id leadin_nx_edge, + + --new destination vertex if exists + ifnull(chtv.vertex_id,0) d_vertex, + + c.max_transport_distance, + ifnull(proc.best_process_id, 0), + --type of new destination vertex if exists + ifnull(chtv.facility_type_id,0) d_vertex_type + + from + (select count(edge_id) parents, + min(miles_travelled) leadin_miles_travelled, + * + from edges + where children_created = 'N' + -----------------do not mess with this "group by" + group by to_node_id, source_facility_id, commodity_id, end_day + ------------------it affects which columns we're checking over for min miles travelled + --------------so that we only get the parent edges we want + order by parents desc + ) p, --parent edges to use in this batch + networkx_edges ch, + networkx_edge_costs nec, + commodities c, + networkx_nodes chfn, + networkx_nodes chtn + left outer join vertices chtv + on (CAST(chtn.location_id as integer) = chtv.location_id + and (p.source_facility_id = chtv.source_facility_id or chtv.source_facility_id = 0) + and (chtv.commodity_id = p.commodity_id or chtv.facility_type_id = {}) + and p.end_day = chtv.schedule_day + and chtv.iob = 'i' + and chtv.storage_vertex = 1) + left outer join candidate_process_commodities proc on + (c.commodity_id = proc.input_commodity and best_process_id is not null) + + + where p.to_node_id = ch.from_node_id + --and p.mode = ch.mode_source --build across modes, control at conservation of flow + and ch.to_node_id = chtn.node_id + and ch.from_node_id = chfn.node_id + and p.phase_of_matter = nec.phase_of_matter_id + and ch.edge_id = nec.edge_id + and ifnull(ch.capacity, 1) > 0 + and p.commodity_id = c.commodity_id + ;""".format(destination_fac_type)) + # --should only get a single leadin edge per networkx/source/commodity/day combination + # leadin edge should be in route_data, current set of min. identifiers + # if i'm trying to add an edge that has an entry in route_data, new miles travelled must be less + + potential_edge_data = potential_edge_data.fetchall() + + main_db_con.execute("update edges set children_created = 'Y' where children_created = 'N';") + + + # this deliberately includes updating "parent" edges that did not get chosen because they weren't the + # current shortest path + # those edges are still "resolved" by this batch + + for row_a in potential_edge_data: + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + mode_oid = row_a[12] + commodity_id = row_a[13] + origin_day = row_a[14] + vertex_id = row_a[15] + source_facility_id = row_a[16] + leadin_edge_miles_travelled = row_a[17] + new_edge_count = row_a[18] + total_route_cost = row_a[19] + leadin_edge_id = row_a[20] + to_vertex = row_a[22] + max_commodity_travel_distance = row_a[23] + input_commodity_process_id = row_a[24] + to_vertex_type = row_a[25] + + # end_day = origin_day + fixed_route_duration + new_miles_travelled = miles + leadin_edge_miles_travelled + if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys()\ + and commodity_mode_dict[mode, commodity_id] == 'Y': + if to_vertex_type ==2: + logger.debug('edge {} goes in to location {} at ' + 'node {} with vertex {}'.format(leadin_edge_id, to_location, to_node, to_vertex)) + + if ((new_miles_travelled > max_commodity_travel_distance and input_commodity_process_id != 0) + or to_vertex_type == destination_fac_type): + #False ): + # designate leadin edge as endcap + children_created = 'E' + destination_yn = 'M' + if to_vertex_type == destination_fac_type: + destination_yn = 'Y' + else: + destination_yn = 'N' + # update the incoming edge to indicate it's an endcap + db_cur.execute( + "update edges set children_created = '{}' where edge_id = {}".format(children_created, + leadin_edge_id)) + if from_location != 'NULL': + db_cur.execute("""insert or ignore into endcap_nodes( + node_id, location_id, mode_source, source_facility_id, commodity_id, process_id, destination_yn) + VALUES ({}, {}, '{}', {}, {},{},'{}'); + """.format(from_node, from_location, mode, source_facility_id, commodity_id, + input_commodity_process_id, destination_yn)) + else: + db_cur.execute("""insert or ignore into endcap_nodes( + node_id, mode_source, source_facility_id, commodity_id, process_id, destination_yn) + VALUES ({}, '{}', {}, {},{},'{}'); + """.format(from_node, mode, source_facility_id, commodity_id, input_commodity_process_id, destination_yn)) + + + # designate leadin edge as endcap + # this does, deliberately, allow endcap status to be overwritten if we've found a shorter + # path to a previous endcap + + # create new edge + elif new_miles_travelled <= max_commodity_travel_distance: + + simple_mode = row_a[3].partition('_')[0] + tariff_id = 0 + if simple_mode == 'pipeline': + + # find tariff_ids + + sql = "select mapping_id from pipeline_mapping where id = {} and id_field_name = " \ + "'source_OID' and source = '{}' and mapping_id is not null;".format(mode_oid, mode) + for tariff_row in db_cur.execute(sql): + tariff_id = tariff_row[0] + + # if there are no edges yet for this day, nx, subc combination, + # AND this is the shortest existing leadin option for this day, nx, subc combination + # we'd be creating an edge for (otherwise wait for the shortest option) + # at this step, some leadin edge should always exist + + if origin_day in range(1, schedule_length + 1): + if (origin_day + fixed_route_duration <= schedule_length): # if link is + # traversable in the timeframe + if simple_mode != 'pipeline' or tariff_id >= 0: + # for allowed commodities + + if from_location == 'NULL' and to_location == 'NULL': + # for each day and commodity, get the corresponding origin vertex id to + # include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough + # than checking if facility type is 'ultimate destination' + # new for bsc, only connect to vertices with matching source_Facility_id + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, + total_route_cost) VALUES ({}, {}, + {}, {}, {}, + {}, {}, {}, + '{}',{},'{}',{}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + # only create edge going into a location if an appropriate vertex exists + elif from_location == 'NULL' and to_location != 'NULL' and to_vertex > 0: + edge_into_facility_counter = edge_into_facility_counter + 1 + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, + total_route_cost) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}', {}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + to_vertex, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + elif from_location != 'NULL' and to_location == 'NULL': + # for each day and commodity, get the corresponding origin vertex id to + # include with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough + # than checking if facility type is 'ultimate destination' + # new for bsc, only connect to vertices with matching source facility id ( + # only limited for RMP vertices) + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, + total_route_cost) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + elif from_location != 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding origin and destination + # vertex ids to include with the edge info + + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, + source_facility_id, + miles_travelled, children_created, edge_count_from_source, + total_route_cost) VALUES ({}, {}, + {}, {}, {}, + {}, {}, + {}, {}, {}, + '{}',{},'{}', {}, + {},'{}',{},'{}', + {}, + {},'{}',{},{}); + """.format(from_node, to_node, + origin_day, origin_day + fixed_route_duration, commodity_id, + vertex_id, to_vertex, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, + miles, simple_mode, tariff_id, phase_of_matter, + source_facility_id, + new_miles_travelled, 'N', new_edge_count, total_route_cost)) + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport' + and children_created in ('N', 'Y', 'E');"""): + source_based_edges_created = row_d[0] + + for row_d in db_cur.execute("""select count(distinct e.edge_id), count(distinct e.nx_edge_id) + from edges e where e.edge_type = 'transport';"""): + transport_edges_created = row_d[0] + nx_edge_count = row_d[1] + + current_edge_data = db_cur.execute("""select count(distinct edge_id), children_created + from edges + where edge_type = 'transport' + group by children_created + order by children_created asc;""") + current_edge_data = current_edge_data.fetchall() + for row in current_edge_data: + if row[1] == 'N': + edges_requiring_children = row[0] + elif row[1] == 'Y': + edges_resolved = row[0] + elif row[1] == 'E': + endcap_edges = row[0] + logger.debug('{} endcap edges designated for candidate generation step'.format(endcap_edges)) + if source_based_edges_created == edges_resolved + endcap_edges: + edges_requiring_children = 0 + if source_based_edges_created == edges_requiring_children + endcap_edges: + edges_resolved = 0 + if source_based_edges_created == edges_requiring_children + edges_resolved: + endcap_edges = 0 + + if counter % 10 == 0 or edges_requiring_children == 0: + logger.info( + '{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( + transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) + counter = counter + 1 + + logger.info('{} transport edges on {} nx edges, created in {} loops, {} edges_requiring_children'.format( + transport_edges_created, nx_edge_count, while_count, edges_requiring_children)) + logger.info("all source-based transport edges created") + + logger.info("create an index for the edges table") + for row in db_cur.execute("select count(*) from endcap_nodes"): + logger.debug("{} endcap nodes before cleanup".format(row)) + + sql = (""" + CREATE INDEX IF NOT EXISTS edge_index ON edges ( + edge_id, route_id, from_node_id, to_node_id, commodity_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id); + """) + db_cur.execute(sql) + return + + +# =============================================================================== + + +def clean_up_endcaps(the_scenario, logger): + logger.info("START: clean_up_endcaps") + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + logger.info("clean up endcap node flagging") + + sql = (""" + drop table if exists e2; + + create table e2 as + select e.edge_id , + e.route_id , + e.from_node_id , + e.to_node_id , + e.start_day , + e.end_day , + e.commodity_id , + e.o_vertex_id , + e.d_vertex_id , + e.max_edge_capacity , + e.volume , + e.capac_minus_volume_zero_floor , + e.min_edge_capacity , + e.capacity_units , + e.units_conversion_multiplier , + e.edge_flow_cost , + e.edge_flow_cost2 , + e.edge_type , + e.nx_edge_id , + e.mode , + e.mode_oid , + e.miles , + e.simple_mode , + e.tariff_id , + e.phase_of_matter , + e.source_facility_id , + e.miles_travelled , + (case when w.cleaned_children = 'C' then w.cleaned_children else e.children_created end) as children_created, + edge_count_from_source , + total_route_cost + from edges e + left outer join + (select 'C' as cleaned_children, t.edge_id + from edges, + (select e.edge_id + from edges e, edges e2, endcap_nodes en + where e.children_created = 'E' + and e.to_node_id = e2.from_node_id + and e.source_facility_id = e2.source_facility_id + and e.commodity_id = e2.commodity_id + and e.miles_travelled < e2.miles_travelled + and e.to_node_id = en.node_id + and e.source_facility_id = en.source_facility_id + and e.commodity_id = en.commodity_id + and en.destination_yn = 'N' + group by e.edge_id + order by e.edge_id + ) t + where edges.edge_id = t.edge_id + and edges.children_created = 'E' + ) w on e.edge_id = w.edge_id + ; + + delete from edges; + + insert into edges select * from e2; + + drop table e2; + """) + logger.info("calling sql on update end caps in the edges table") + db_cur.executescript(sql) + + logger.info("clean up the endcap_nodes table") + + sql = (""" + drop table if exists en_clean; + + create table en_clean + as select en.* + from endcap_nodes en, edges e + where e.to_node_id = en.node_id + and e.commodity_id = en.commodity_id + and e.source_facility_id = en.source_facility_id + and e.children_created = 'E' + ; + + drop table endcap_nodes; + create table endcap_nodes as select * from en_clean; + + drop table en_clean; + + """) + logger.info("calling sql to clean up the endcap_nodes table") + db_cur.executescript(sql) + + for row in db_cur.execute("select count(*) from endcap_nodes;"): + logger.debug("{} endcap nodes after cleanup".format(row)) + + return + + +# =============================================================================== + + +def generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_length, logger): + logger.info("START: generate_all_edges_without_max_commodity_constraint") + # make sure this covers edges from an RMP if the commodity has no max transport distance + + multi_commodity_name = "multicommodity" + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + db_cur = main_db_con.cursor() + + commodity_mode_data = main_db_con.execute("select * from commodity_mode;") + commodity_mode_data = commodity_mode_data.fetchall() + commodity_mode_dict = {} + for row in commodity_mode_data: + mode = row[0] + commodity_id = int(row[1]) + commodity_phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + commodity_mode_dict[mode, commodity_id] = allowed_yn + + counter = 0 + + # for all commodities with no max transport distance or output by a candidate process + source_facility_id = 0 + + # create transport edges, only between storage vertices and nodes, based on networkx graph + # never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link + # iterate through nx edges: create 1 edge per viable commodity as allowed by nx phase_of_matter or allowed + # commodities. Should also be per day, subject to nx edge schedule + # if there is a max transport distance on a candidate process output, disregard for now because we don't know + # where the candidates will be located + + # select data for transport edges + + sql = """select + ne.edge_id, + ifnull(fn.location_id, 'NULL'), + ifnull(tn.location_id, 'NULL'), + ne.mode_source, + ifnull(nec.phase_of_matter_id, 'NULL'), + nec.route_cost, + ne.from_node_id, + ne.to_node_id, + nec.dollar_cost, + ne.miles, + ne.capacity, + ne.artificial, + ne.mode_source_oid + from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec + + where ne.from_node_id = fn.node_id + and ne.to_node_id = tn.node_id + and ne.edge_id = nec.edge_id + and ifnull(ne.capacity, 1) > 0 + ;""" + nx_edge_data = main_db_con.execute(sql) + nx_edge_data = nx_edge_data.fetchall() + for row_a in nx_edge_data: + + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + mode_oid = row_a[12] + simple_mode = row_a[3].partition('_')[0] + + counter = counter + 1 + + tariff_id = 0 + if simple_mode == 'pipeline': + + # find tariff_ids + sql = "select mapping_id from pipeline_mapping where id = {} and id_field_name = 'source_OID' and " \ + "source = '{}' and mapping_id is not null;".format(mode_oid, mode) + for tariff_row in db_cur.execute(sql): + tariff_id = tariff_row[0] + + if mode in the_scenario.permittedModes and (mode, commodity_id) in commodity_mode_dict.keys()\ + and commodity_mode_dict[mode, commodity_id] == 'Y': + + # Edges are placeholders for flow variables + # if both ends have no location, iterate through viable commodities and days, create edge + # for all days (restrict by link schedule if called for) + # for all allowed commodities, as currently defined by link phase of matter + for day in range(1, schedule_length + 1): + if (day + fixed_route_duration <= schedule_length): # if link is traversable in the + # timeframe + if simple_mode != 'pipeline' or tariff_id >= 0: + # for allowed commodities that can be output by some facility or process in the scenario + for row_c in db_cur.execute("""select commodity_id + from source_commodity_ref + where phase_of_matter = '{}' + and max_transport_distance_flag = 'N' + group by commodity_id + union + select commodity_id + from candidate_process_commodities + where phase_of_matter = '{}' + and io = 'o' + group by commodity_id""".format(phase_of_matter, phase_of_matter)): + db_cur4 = main_db_con.cursor() + commodity_id = row_c[0] + # source_facility_id = row_c[1] # fixed to 0 for all edges created by this method + + if from_location == 'NULL' and to_location == 'NULL': + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + + elif from_location != 'NULL' and to_location == 'NULL': + # for each day and commodity, get the corresponding origin vertex id to include + # with the edge info + # origin vertex must not be "ultimate_destination + # transport link outgoing from facility - checking fc.io is more thorough than + # checking if facility type is 'ultimate destination' + # new for bsc, only connect to vertices with matching source_facility_id (only + # limited for RMP vertices) + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and + v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): + from_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + from_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + elif from_location == 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding destination vertex id to + # include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and + v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): + to_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + to_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + elif from_location != 'NULL' and to_location != 'NULL': + # for each day and commodity, get the corresponding origin and destination vertex + # ids to include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and + v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, source_facility_id)): + from_vertex_id = row_d[0] + db_cur5 = main_db_con.cursor() + for row_e in db_cur5.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and + v.source_facility_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, source_facility_id)): + to_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, + to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, + phase_of_matter, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, {}, + {}, {}, {}, + '{}',{},'{}', {},{},'{}',{},'{}',{}); + """.format(from_node, to_node, + day, day + fixed_route_duration, commodity_id, + from_vertex_id, to_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, + tariff_id, phase_of_matter, source_facility_id)) + + logger.debug("all transport edges created") + + logger.info("all edges created") + logger.info("create an index for the edges table by nodes") + index_start_time = datetime.datetime.now() + sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( + edge_id, route_id, from_node_id, to_node_id, commodity_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, source_facility_id);""") + db_cur.execute(sql) + logger.info("edge_index Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(index_start_time))) + return + + +# =============================================================================== + + +def pre_setup_pulp(logger, the_scenario): + logger.info("START: pre_setup_pulp for candidate generation step") + from ftot_pulp import commodity_mode_setup + + commodity_mode_setup(the_scenario, logger) + + check_max_transport_distance_for_OC_step(the_scenario, logger) + + source_as_subcommodity_setup(the_scenario, logger) + + from ftot_pulp import generate_schedules + logger.debug("----- Using generate_all_vertices method imported from ftot_pulp ------") + schedule_dict, schedule_length = generate_schedules(the_scenario, logger) + + # Re-create schedule dictionary with one-day schedules with availability = average availability + schedule_avg = schedule_avg_availabilities(the_scenario, schedule_dict, schedule_length, logger) + schedule_avg_length = 1 + + from ftot_pulp import generate_all_vertices + logger.debug("----- Using generate_all_vertices method imported from ftot_pulp ------") + generate_all_vertices(the_scenario, schedule_avg, schedule_avg_length, logger) + + from ftot_pulp import add_storage_routes + logger.debug("----- Using add_storage_routes method imported from ftot_pulp ------") + add_storage_routes(the_scenario, logger) + + from ftot_pulp import generate_connector_and_storage_edges + logger.debug("----- Using generate_connector_and_storage_edges method imported from ftot_pulp ------") + generate_connector_and_storage_edges(the_scenario, logger) + + from ftot_pulp import generate_first_edges_from_source_facilities + logger.debug("----- Using generate_first_edges_from_source_facilities method imported from ftot_pulp ------") + generate_first_edges_from_source_facilities(the_scenario, schedule_avg_length, logger) + + generate_all_edges_from_source_facilities(the_scenario, schedule_avg_length, logger) + + clean_up_endcaps(the_scenario, logger) + + generate_all_edges_without_max_commodity_constraint(the_scenario, schedule_avg_length, logger) + + logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) + + from ftot_pulp import set_edges_volume_capacity + logger.debug("----- Using set_edges_volume_capacity method imported from ftot_pulp ------") + set_edges_volume_capacity(the_scenario, logger) + + return + + +# =============================================================================== + + +def create_flow_vars(the_scenario, logger): + logger.info("START: create_flow_vars") + + # call helper method to get list of unique IDs from the Edges table. + # use a the rowid as a simple unique integer index + edge_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, source_facility_id + from edges;""") + edge_list_data = edge_list_cur.fetchall() + counter = 0 + for row in edge_list_data: + if counter % 500000 == 0: + logger.info( + "processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) + counter += 1 + # create an edge for each commodity allowed on this link - this construction may change as specific + # commodity restrictions are added + edge_list.append((row[0])) + + # flow_var is the flow on each arc, being determined; this can be defined any time after all_arcs is defined + flow_var = LpVariable.dicts("Edge", edge_list, 0, None) + logger.detailed_debug("DEBUG: Size of flow_var: {:,.0f}".format(sys.getsizeof(flow_var))) + + return flow_var + + +# =============================================================================== + + +def create_unmet_demand_vars(the_scenario, logger): + logger.info("START: create_unmet_demand_vars") + demand_var_list = [] + # may create vertices with zero demand, but only for commodities that the facility has demand for at some point + #checks material incoming from storage vertex + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) + top_level_commodity_name, v.udp + from vertices v, commodities c, facility_type_id ft, facilities f + where v.commodity_id = c.commodity_id + and ft.facility_type = "ultimate_destination" + and v.storage_vertex = 0 + and v.facility_type_id = ft.facility_type_id + and v.facility_id = f.facility_id + and f.ignore_facility = 'false' + group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) + ;""".format('')): + # facility_id, day, and simplified commodity name, udp + demand_var_list.append((row[0], row[1], row[2], row[3])) + + unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) + + return unmet_demand_var + + +# =============================================================================== + + +def create_candidate_processor_build_vars(the_scenario, logger): + logger.info("START: create_candidate_processor_build_vars") + processors_build_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute( + "select f.facility_id from facilities f, facility_type_id ft where f.facility_type_id = " + "ft.facility_type_id and facility_type = 'processor' and candidate = 1 and ignore_facility = 'false' " + "group by facility_id;"): + # grab all candidate processor facility IDs + processors_build_list.append(row[0]) + + processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list, 0, None, 'Binary') + + return processor_build_var + + +# =============================================================================== + + +def create_binary_processor_vertex_flow_vars(the_scenario, logger): + logger.info("START: create_binary_processor_vertex_flow_vars") + processors_flow_var_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 0 + group by v.facility_id, v.schedule_day;"""): + # facility_id, day + processors_flow_var_list.append((row[0], row[1])) + + processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0, None, 'Binary') + + return processor_flow_var + + +# =============================================================================== + + +def create_processor_excess_output_vars(the_scenario, logger): + logger.info("START: create_processor_excess_output_vars") + excess_var_list = [] + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + xs_cur = db_cur.execute(""" + select vertex_id, commodity_id + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 1;""") + # facility_id, day, and simplified commodity name + xs_data = xs_cur.fetchall() + for row in xs_data: + excess_var_list.append(row[0]) + + excess_var = LpVariable.dicts("XS", excess_var_list, 0, None) + + return excess_var + + +# =============================================================================== + + +def create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars): + logger.debug("START: create_opt_problem") + prob = LpProblem("Flow assignment", LpMinimize) + + logger.detailed_debug("DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_vars))) + logger.detailed_debug("DEBUG: length of flow_vars: {}".format(len(flow_vars))) + logger.detailed_debug("DEBUG: length of processor_build_vars: {}".format(len(processor_build_vars))) + + unmet_demand_costs = [] + flow_costs = {} + logger.detailed_debug("DEBUG: start loop through sql to append unmet_demand_costs") + for u in unmet_demand_vars: + udp = u[3] + unmet_demand_costs.append(udp * unmet_demand_vars[u]) + logger.detailed_debug("DEBUG: finished loop through sql to append unmet_demand_costs. total records: {}".format( + len(unmet_demand_costs))) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + logger.detailed_debug("DEBUG: start sql execute to get flow cost data") + # Flow cost memory improvements: only get needed data; dict instead of list; narrow in lpsum + flow_cost_var = db_cur.execute("select edge_id, edge_flow_cost from edges e group by edge_id;") + logger.detailed_debug("DEBUG: start the fetchall") + flow_cost_data = flow_cost_var.fetchall() + logger.detailed_debug( + "DEBUG: start iterating through {:,.0f} flow_cost_data records".format(len(flow_cost_data))) + counter = 0 + for row in flow_cost_data: + edge_id = row[0] + edge_flow_cost = row[1] + counter += 1 + + # flow costs cover transportation and storage + flow_costs[edge_id] = edge_flow_cost + logger.detailed_debug( + "DEBUG: finished loop through sql to append flow costs: total records: {:,.0f}".format(len(flow_costs))) + + logger.detailed_debug("debug: start prob+= unmet_demand_costs + flow cost + no processor build costs in" + "candidate generation step") + prob += (lpSum(unmet_demand_costs) + lpSum( + flow_costs[k] * flow_vars[k] for k in flow_costs)), "Total Cost of Transport, storage, and penalties" + logger.detailed_debug("debug: done prob+= unmet_demand_costs + flow cost + no processor build costs in " + "candidate generation step") + + logger.debug("FINISHED: create_opt_problem") + return prob + + +# =============================================================================== + + +def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): + logger.debug("START: create_constraint_unmet_demand") + + # apply activity_level to get corresponding actual demand for var + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + # var has form(facility_name, day, simple_fuel) + # unmet demand commodity should be simple_fuel = supertype + logger.detailed_debug("DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_var))) + + demand_met_dict = defaultdict(list) + actual_demand_dict = {} + + # want to specify that all edges leading into this vertex + unmet demand = total demand + # demand primary (non-storage) vertices + + db_cur = main_db_con.cursor() + # each row_a is a primary vertex whose edges in contributes to the met demand of var + # will have one row for each fuel subtype in the scenario + unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + from vertices v, commodities c, facility_type_id ft, facilities f, edges e + where v.facility_id = f.facility_id + and ft.facility_type = 'ultimate_destination' + and f.facility_type_id = ft.facility_type_id + and f.ignore_facility = 'false' + and v.facility_type_id = ft.facility_type_id + and v.storage_vertex = 0 + and c.commodity_id = v.commodity_id + and e.d_vertex_id = v.vertex_id + group by v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.source_facility_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + ;""") + + unmet_data = unmet_data.fetchall() + for row_a in unmet_data: + var_full_demand = row_a[2] + proportion_of_supertype = row_a[3] + var_activity_level = row_a[4] + facility_id = row_a[6] + day = row_a[7] + top_level_commodity = row_a[8] + udp = row_a[9] + edge_id = row_a[10] + var_actual_demand = var_full_demand * var_activity_level + + # next get inbound edges and apply appropriate modifier proportion to get how much of var's demand they + # satisfy + demand_met_dict[(facility_id, day, top_level_commodity, udp)].append( + flow_var[edge_id] * proportion_of_supertype) + actual_demand_dict[(facility_id, day, top_level_commodity, udp)] = var_actual_demand + + for key in unmet_demand_var: + if key in demand_met_dict: + # then there are some edges in + prob += lpSum(demand_met_dict[key]) == actual_demand_dict[key] - unmet_demand_var[ + key], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(key[0], + key[1], + key[2]) + else: + if key not in actual_demand_dict: + pdb.set_trace() + # no edges in, so unmet demand equals full demand + prob += actual_demand_dict[key] == unmet_demand_var[ + key], "constraint set unmet demand variable for facility {}, day {}, commodity {} - " \ + "no edges able to meet demand".format( + key[0], key[1], key[2]) + + logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") + return prob + + +# =============================================================================== + + +def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + # force flow out of origins to be <= supply + + # for each primary (non-storage) supply vertex + # flow out of a vertex <= supply of the vertex, true for every day and commodity + # Assumption - each RMP produces a single commodity + # Assumption - only one vertex exists per day per RMP (no multi commodity or subcommodity) + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row_a in db_cur.execute("""select vertex_id, activity_level, supply + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and ft.facility_type = 'raw_material_producer' + and storage_vertex = 0;"""): + supply_vertex_id = row_a[0] + activity_level = row_a[1] + max_daily_supply = row_a[2] + actual_vertex_supply = activity_level * max_daily_supply + + flow_out = [] + db_cur2 = main_db_con.cursor() + # select all edges leaving that vertex and sum their flows + # should be a single connector edge + for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): + edge_id = row_b[0] + flow_out.append(flow_var[edge_id]) + + prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format( + actual_vertex_supply, supply_vertex_id) + + logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") + return prob + + +# =============================================================================== + + +def create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_var): + logger.info("STARTING: create_primary_processor_vertex_constraints - capacity and conservation of flow") + # for all of these vertices, flow in always == flow out + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + # total flow in == total flow out, subject to conversion; + # summed over commodities but not days; does not check commodity proportions + # dividing by "required quantity" functionally converts all commodities to the same "processor-specific units" + + # processor primary vertices with input commodity and quantity needed to produce specified output quantities + # 2 sets of constraints; one for the primary processor vertex to cover total flow in and out + # one for each input and output commodity (sum over sources) to ensure its ratio matches facility_commodities + + logger.debug("conservation of flow and commodity ratios, primary processor vertices:") + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' + end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 + end) constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + ifnull(f.candidate, 0) candidate_check, + e.source_facility_id, + v.source_facility_id, + v.commodity_id + from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f + join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) + where ft.facility_type = 'processor' + and v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 0 + and v.facility_id = fc.facility_id + and fc.commodity_id = c.commodity_id + and fc.commodity_id = e.commodity_id + group by v.vertex_id, + in_or_out_edge, + constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + candidate_check, + e.source_facility_id, + v.commodity_id, + v.source_facility_id + order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id + ;""" + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + sql_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info( + "execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t " + "".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + sql_data = sql_data.fetchall() + logger.info( + "fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + # Nested dictionaries + # flow_in_lists[primary_processor_vertex_id] = dict of commodities handled by that processor vertex + + # flow_in_lists[primary_processor_vertex_id][commodity1] = + # list of edge ids that flow that commodity into that vertex + + # flow_in_lists[vertex_id].values() to get all flow_in edges for all commodities, a list of lists + + flow_in_lists = {} + flow_out_lists = {} + + for row_a in sql_data: + + vertex_id = row_a[0] + in_or_out_edge = row_a[1] + commodity_id = row_a[3] + edge_id = row_a[5] + quantity = float(row_a[7]) + + if in_or_out_edge == 'in': + flow_in_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it + flow_in_lists[vertex_id].setdefault((commodity_id, quantity), []).append(flow_var[edge_id]) + # flow_in_lists[vertex_id] is itself a dict keyed on commodity and quantity; + # value is a list of edge ids into that vertex of that commodity + + elif in_or_out_edge == 'out': + flow_out_lists.setdefault(vertex_id, {}) # if the vertex isn't in the main dict yet, add it + flow_out_lists[vertex_id].setdefault((commodity_id, quantity), []).append(flow_var[edge_id]) + + # Because we keyed on commodity, source facility tracking is merged as we pass through the processor vertex + + # 1) for each output commodity, check against an input to ensure correct ratio - only need one input + # 2) for each input commodity, check against an output to ensure correct ratio - only need one output; + # 2a) first sum sub-flows over input commodity + # 3)-x- handled by daily processor capacity constraint [calculate total processor input to ensure + # compliance with capacity constraint + + # 1---------------------------------------------------------------------- + + for key, value in iteritems(flow_out_lists): + vertex_id = key + zero_in = False + if vertex_id in flow_in_lists: + compare_input_list = [] + in_quantity = 0 + in_commodity_id = 0 + for ikey, ivalue in iteritems(flow_in_lists[vertex_id]): + in_commodity_id = ikey[0] + in_quantity = ikey[1] + # edge_list = value2 + compare_input_list = ivalue + else: + zero_in = True + + # value is a dict - we loop once here for each output commodity at the vertex + for key2, value2 in iteritems(value): + out_commodity_id = key2[0] + out_quantity = key2[1] + # edge_list = value2 + flow_var_list = value2 + if zero_in: + prob += lpSum( + flow_var_list) == 0, "processor flow, vertex {} has zero in so zero out of commodity {}".format( + vertex_id, out_commodity_id) + else: + # ratio constraint for this output commodity relative to total input + required_flow_out = lpSum(flow_var_list) / out_quantity + # check against an input dict + prob += required_flow_out == lpSum( + compare_input_list) / in_quantity, "processor flow, vertex {}, commodity {} output quantity " \ + "checked against commodity {} input quantity".format( + vertex_id, out_commodity_id, in_commodity_id) + + # 2---------------------------------------------------------------------- + for key, value in iteritems(flow_in_lists): + vertex_id = key + zero_out = False + if vertex_id in flow_out_lists: + compare_output_list = [] + out_quantity = 0 + for okey, ovalue in iteritems(flow_out_lists[vertex_id]): + out_commodity_id = okey[0] + out_quantity = okey[1] + # edge_list = value2 + compare_output_list = ovalue + else: + zero_out = True + + # value is a dict - we loop once here for each input commodity at the vertex + for key2, value2 in iteritems(value): + in_commodity_id = key2[0] + in_quantity = key2[1] + # edge_list = value2 + flow_var_list = value2 + if zero_out: + prob += lpSum( + flow_var_list) == 0, "processor flow, vertex {} has zero out so zero in of commodity {}".format( + vertex_id, in_commodity_id) + else: + # ratio constraint for this output commodity relative to total input + required_flow_in = lpSum(flow_var_list) / in_quantity + # check against an input dict + prob += required_flow_in == lpSum( + compare_output_list) / out_quantity, "processor flow, vertex {}, commodity {} input quantity " \ + "checked against commodity {} output quantity".format( + vertex_id, in_commodity_id, out_commodity_id) + + logger.debug("FINISHED: create_primary_processor_conservation_of_flow_constraints") + return prob + + +# =============================================================================== + + +def create_constraint_conservation_of_flow_storage_vertices(logger, the_scenario, prob, flow_var, + processor_excess_vars): + logger.debug("STARTING: create_constraint_conservation_of_flow_storage_vertices") + storage_vertex_constraint_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + logger.info("conservation of flow, storage vertices:") + # storage vertices, any facility type + # these have at most one direction of transport edges, so no need to track mode + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' + end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 + end) constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id, v.facility_id, c.commodity_name, + v.activity_level, + ft.facility_type + + from vertices v, facility_type_id ft, commodities c, facilities f + join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) and (e.o_vertex_id = + v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) + + where v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 1 + and v.commodity_id = c.commodity_id + + group by v.vertex_id, + in_or_out_edge, + constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id,v.facility_id, c.commodity_name, + v.activity_level + + order by v.facility_id, v.vertex_id, e.edge_id + ;""" + + # get the data from sql and see how long it takes. + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + vertexid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + vertexid_data = vertexid_data.fetchall() + logger.info("fetchall storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_in_lists = {} + flow_out_lists = {} + for row_v in vertexid_data: + vertex_id = row_v[0] + in_or_out_edge = row_v[1] + constraint_day = row_v[2] + commodity_id = row_v[3] + edge_id = row_v[4] + facility_type = row_v[9] + + if in_or_out_edge == 'in': + flow_in_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( + flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((vertex_id, commodity_id, constraint_day, facility_type), []).append( + flow_var[edge_id]) + + logger.info("adding processor excess variables to conservation of flow") + for key, value in iteritems(flow_out_lists): + vertex_id = key[0] + facility_type = key[3] + if facility_type == 'processor': + flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) + + for key, value in iteritems(flow_out_lists): + + if key in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum( + flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + else: + prob += lpSum(flow_out_lists[key]) == lpSum( + 0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + for key, value in iteritems(flow_in_lists): + + if key not in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum( + 0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], + key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + logger.info("total conservation of flow constraints created on storage vertices: {}".format( + storage_vertex_constraint_counter)) + + return prob + + +# =============================================================================== + +def create_constraint_conservation_of_flow_endcap_nodes(logger, the_scenario, prob, flow_var, + processor_excess_vars): + # This creates constraints for all non-vertex nodes, with variant rules for endcaps nodes + logger.debug("STARTING: create_constraint_conservation_of_flow_endcap_nodes") + node_constraint_counter = 0 + passthrough_constraint_counter = 0 + other_endcap_constraint_counter = 0 + endcap_no_reverse_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + logger.info("conservation of flow, nx_nodes, with endcaps:") + # non-vertex nodes, no facility type, connect 2 transport edges over matching days and commodities + # for each day, get all edges in and out of the node. Sort them by commodity and whether they're going in or + # out of the node + + # retrieve candiate process information + # process_dict[input commodity id] = list [proc_id, quantity, tuples (output comm. id, quantity)] + # initialze Value as list containing process id, then add outputs + process_dict = {} + process_outputs_dict = {} + + # list each output with its input to key off; don't use process id + # nested dictionary: first key is input commodity id + # second key is output commodity id; value is list with preferred process: + # # process_id,input quant, output quantity, output ratio, with the largest output ratio for that commodity pair + # list each output with its input to key on + sql = """select process_id, commodity_id, quantity + from candidate_process_commodities + where io = 'i';""" + process_inputs = db_cur.execute(sql) + process_inputs = process_inputs.fetchall() + + for row in process_inputs: + process_id = row[0] + commodity_id = row[1] + quantity = row[2] + process_dict.setdefault(commodity_id, [process_id, quantity]) + process_dict[commodity_id] = [process_id, quantity] + + + sql = """select o.process_id, o.commodity_id, o.quantity, i.commodity_id + from candidate_process_commodities o, candidate_process_commodities i + where o.io = 'o' + and i.process_id = o.process_id + and i.io = 'i';""" + process_outputs = db_cur.execute(sql) + process_outputs = process_outputs.fetchall() + + for row in process_outputs: + process_id = row[0] + output_commodity_id = row[1] + quantity = row[2] + input_commodity_id = row[3] + + process_dict[input_commodity_id].append((output_commodity_id, quantity)) + process_outputs_dict.setdefault(process_id, []).append(output_commodity_id) + + sql = """select nn.node_id, + (case when e.from_node_id = nn.node_id then 'out' when e.to_node_id = nn.node_id then 'in' else 'error' + end) in_or_out_edge, + (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 + end) constraint_day, + e.commodity_id, + ifnull(mode, 'NULL'), + e.edge_id, nx_edge_id, + miles, + (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, + e.source_facility_id, + e.commodity_id, + (case when en.node_id is null then 'N' else 'E' end) as endcap, + (case when en.node_id is null then 0 else en.process_id end) as endcap_process, + (case when en.node_id is null then 0 else en.source_facility_id end) as endcap_source_facility + from networkx_nodes nn + join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) + left outer join endcap_nodes en on (nn.node_id = en.node_id and e.commodity_id = en.commodity_id and + e.source_facility_id = en.source_facility_id) + where nn.location_id is null + order by nn.node_id, e.commodity_id, + (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 + end), + in_or_out_edge, e.source_facility_id, e.commodity_id + ;""" + + # get the data from sql and see how long it takes. + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + nodeid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info( + "execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t " + "".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + nodeid_data = nodeid_data.fetchall() + logger.info( + "fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_in_lists = {} + flow_out_lists = {} + endcap_ref = {} + + for row_a in nodeid_data: + node_id = row_a[0] + in_or_out_edge = row_a[1] + constraint_day = row_a[2] + commodity_id = row_a[3] + mode = row_a[4] + edge_id = row_a[5] + nx_edge_id = row_a[6] + miles = row_a[7] + intermodal = row_a[8] + source_facility_id = row_a[9] + commodity_id = row_a[10] + endcap_flag = row_a[11] # is 'E' if endcap, else 'Y' or 'null' + endcap_input_process = row_a[12] + endcap_source_facility = row_a[13] + # endcap ref is keyed on node_id, makes a list of [commodity_id, list, list]; + # first list will be source facilities + if endcap_source_facility > 0: + endcap_ref.setdefault(node_id, [endcap_input_process, commodity_id, [], []]) + if endcap_source_facility not in endcap_ref[node_id][2]: + endcap_ref[node_id][2].append(endcap_source_facility) + + # if node is not intermodal, conservation of flow holds per mode; + # if intermodal, then across modes + # if node is endcap, conservation of flow holds across source facilities + if intermodal == 'N': + if in_or_out_edge == 'in': + flow_in_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id, + endcap_input_process, mode), []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((node_id, intermodal, source_facility_id, constraint_day, commodity_id, + endcap_input_process, mode), []).append(flow_var[edge_id]) + else: + if in_or_out_edge == 'in': + flow_in_lists.setdefault( + (node_id, intermodal, source_facility_id, constraint_day, commodity_id, endcap_input_process), + []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault( + (node_id, intermodal, source_facility_id, constraint_day, commodity_id, endcap_input_process), + []).append(flow_var[edge_id]) + + endcap_dict = {} + + for node, value in iteritems(endcap_ref): + # add output commodities to endcap_ref[node][3] + # endcap_ref[node][2] is a list of source facilities this endcap matches for the input commodity + process_id = value[0] + if process_id > 0: + endcap_ref[node][3] = process_outputs_dict[process_id] + + # if this node has at least one edge flowing out + for key, value in iteritems(flow_in_lists): + + if node_constraint_counter < 50000: + if node_constraint_counter % 5000 == 0: + logger.info("flow constraint created for {} nodes".format(node_constraint_counter)) + # logger.info('{} edges constrained'.format(edge_counter) + + if node_constraint_counter % 50000 == 0: + if node_constraint_counter == 50000: + logger.info("5000 nodes constrained, switching to log every 50000 instead of 5000") + logger.info("flow constraint created for {} nodes".format(node_constraint_counter)) + # logger.info('{} edges constrained'.format(edge_counter)) + + node_id = key[0] + intermodal_flag = key[1] + source_facility_id = key[2] + day = key[3] + commodity_id = key[4] + endcap_input_process = key[5] + + if len(key) == 7: + node_mode = key[6] + else: + node_mode = 'intermodal' + + # if this node is an endcap + # if this commodity and source match the inputs for the endcap process + # or if this output commodity matches an input process for this node, enter the first loop + # if output commodity, don't do anything - will be handled via input commodity + if (node_id in endcap_ref and + ((endcap_input_process == endcap_ref[node_id][0] and commodity_id == endcap_ref[node_id][ + 1] and source_facility_id in endcap_ref[node_id][2]) + or commodity_id in endcap_ref[node_id][3])): + # if we need to handle this commodity & source according to the endcap process + # and it is an input or output of the current process + # if this is an output of a different process, it fails the above "if" and gets a standard constraint + # below + if commodity_id in process_dict and (node_id, day) not in endcap_dict: + + # tracking which endcap nodes we've already made constraints for + endcap_dict[node_id, day] = commodity_id + + # else, this is an output of this candidate process, and it's endcap constraint is created by the + # input commodity + # which must have edges in, or this would never have been tagged as an endcap node as long as the + # input commodity has some flow in + + # for this node, on this day, with this source facility - intermodal and mode and encapflag + # unchanged, commodity may not + # for endcap nodes, can't check for flow in exactly the same way because commodity may change + input_quantity = process_dict[commodity_id][1] + # since we're working with input to an endcap, need all matching inputs, which may include other + # source facilities as specified in endcap_ref[node_id][2] + in_key_list = [key] + if len(endcap_ref[node_id][2]) > 1: + for alt_source in endcap_ref[node_id][2]: + if alt_source != source_facility_id: + + if intermodal_flag == 'N': + new_key = ( + node_id, intermodal_flag, alt_source, day, commodity_id, endcap_input_process, + node_mode) + else: + new_key = ( + node_id, intermodal_flag, alt_source, day, commodity_id, endcap_input_process) + + in_key_list.append(new_key) + + # this is an endcap for this commodity and source, so unprocessed flow out is not allowed + # unless it's from a different source + + prob += lpSum(flow_out_lists[k] for k in in_key_list if k in flow_out_lists) == lpSum( + 0), "conservation of flow (zero allowed out, must be processed), endcap, nx node {}, " \ + "source facility {}, commodity {}, day {}, mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + + outputs_dict = {} + # starting with the 3rd item in the list, the first output commodity tuple + for i in range(2, len(process_dict[commodity_id])): + output_commodity_id = process_dict[commodity_id][i][0] + output_quantity = process_dict[commodity_id][i][1] + outputs_dict[output_commodity_id] = output_quantity + # now we have a dict of allowed outputs for this input; construct keys for matching flows + # allow cross-modal flow?no, these are not candidates yet, want to use exists routes + + for o_commodity_id, quant in iteritems(outputs_dict): + # creating one constraint per output commodity, per endcap node + node_constraint_counter = node_constraint_counter + 1 + output_source_facility_id = 0 + output_process_id = 0 + + # setting up the outflow edges to check + if node_mode == 'intermodal': + out_key = ( + node_id, intermodal_flag, output_source_facility_id, day, o_commodity_id, + output_process_id) + else: + out_key = ( + node_id, intermodal_flag, output_source_facility_id, day, o_commodity_id, + output_process_id, node_mode) + + if out_key not in flow_out_lists: + # node has edges flowing in, but no valid out edges; restrict flow in to zero + prob += lpSum(flow_in_lists[k] for k in in_key_list if k in flow_in_lists) == lpSum( + 0), "conservation of flow (zero allowed in), endcap, nx node {}, source facility {}, " \ + "commodity {}, day {}, mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + other_endcap_constraint_counter = other_endcap_constraint_counter + 1 + else: + # out_key is in flow_out_lists + # endcap functioning as virtual processor + # now aggregate flows for this commodity + # processsed_flow_out = lpSum(flow_out_lists[out_key])/quant + # divide by /input_quantity for the required processing ratio + + # if key is in flow_out_lists, some input material may pass through unprocessed + # if out_key is in flow_in_lists, some output material may pass through unprocessed + # difference is the amount of input material processed: lpSum(flow_in_lists[key]) - + # lpSum(flow_out_lists[key]) + agg_inflow_lists = [] + for k in in_key_list: + if k in flow_in_lists: + for l in flow_in_lists[k]: + if l not in agg_inflow_lists: + agg_inflow_lists.append(l) + + if out_key in flow_in_lists: + prob += lpSum(agg_inflow_lists) / input_quantity == ( + lpSum(flow_out_lists[out_key]) - lpSum(flow_in_lists[out_key])) / quant, \ + "conservation of flow, endcap as processor, passthrough processed,nx node {}," \ + "source facility {}, in commodity {}, out commodity {}, day {},mode {}".format( + node_id, source_facility_id, commodity_id, o_commodity_id, day, node_mode) + passthrough_constraint_counter = passthrough_constraint_counter + 1 + else: + prob += lpSum(agg_inflow_lists) / input_quantity == lpSum(flow_out_lists[ + out_key]) / quant, \ + "conservation of flow, endcap as processor, no passthrough, nx node {}, " \ + "source facility {}, in commodity {}, out commodity {}, day {}, " \ + "mode {}".format( + node_id, source_facility_id, commodity_id, o_commodity_id, day, + node_mode) + other_endcap_constraint_counter = other_endcap_constraint_counter + 1 + + + + + + else: + # if this node is not an endcap, or has no viable candidate process, handle as standard + # if this node has at least one edge flowing in, and out, ensure flow is compliant + # here is where we need to add the processor ratios and allow conversion, for endcaps + if key in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum(flow_out_lists[ + key]), "conservation of flow, nx node {}, " \ + "source facility {}, commodity {}, day {}, " \ + "mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + # node has edges flowing in, but not out; restrict flow in to zero + else: + prob += lpSum(flow_in_lists[key]) == lpSum( + 0), "conservation of flow (zero allowed in), nx node {}, source facility {}, commodity {}, " \ + "day {}, mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + for key, value in iteritems(flow_out_lists): + node_id = key[0] + source_facility_id = key[2] + day = key[3] + commodity_id = key[4] + if len(key) == 7: + node_mode = key[6] + else: + node_mode = 'intermodal' + + # node has edges flowing out, but not in; restrict flow out to zero + if key not in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum( + 0), "conservation of flow (zero allowed out), nx node {}, source facility {}, commodity {}, " \ + "day {}, mode {}".format( + node_id, source_facility_id, commodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + logger.info( + "total non-endcap conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) + + # Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints + + logger.debug("endcap constraints: passthrough {}, other {}, total no reverse {}".format( + passthrough_constraint_counter, other_endcap_constraint_counter, endcap_no_reverse_counter)) + logger.info("FINISHED: create_constraint_conservation_of_flow_endcap_nodes") + + return prob + + +# =============================================================================== + + +def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_pipeline_capacity") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) + logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + # capacity for pipeline tariff routes + # with source tracking, may have multiple flows per segment, slightly diff commodities + sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow + allowed_flow, l.source, e.mode, instr(e.mode, l.source) + from edges e, pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where e.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(e.mode, l.source)>0 + group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source + ;""" + # capacity needs to be shared over link_id for any edge_id associated with that link + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + pipeline_capac_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for transport edges:") + logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + pipeline_capac_data = pipeline_capac_data.fetchall() + logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format( + get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in pipeline_capac_data: + edge_id = row_a[0] + tariff_id = row_a[1] + link_id = row_a[2] + # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall + # Link capacity * 42 is now in kgal per day, to match flow in kgal + link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[3] + start_day = row_a[4] + capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS * row_a[5], 0) + min_restricted_capacity = max(capac_minus_background_flow_kgal, + link_capacity_kgal_per_day * the_scenario.minCapacityLevel) + + capacity_nodes_mode_source = row_a[6] + edge_mode = row_a[7] + mode_match_check = row_a[8] + if 'pipeline' in the_scenario.backgroundFlowModes: + link_use_capacity = min_restricted_capacity + else: + link_use_capacity = link_capacity_kgal_per_day + + # add flow from all relevant edges, for one start; may be multiple tariffs + flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format( + key[0], key[3], key[2]) + + logger.debug("pipeline capacity constraints created for all transport routes") + + logger.debug("FINISHED: create_constraint_pipeline_capacity") + return prob + + +# =============================================================================== + + +def setup_pulp_problem_candidate_generation(the_scenario, logger): + logger.info("START: setup PuLP problem") + + # flow_var is the flow on each edge by commodity and day. + # the optimal value of flow_var will be solved by PuLP + flow_vars = create_flow_vars(the_scenario, logger) + + # unmet_demand_var is the unmet demand at each destination, being determined + unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) + + # processor_build_vars is the binary variable indicating whether a candidate processor is used and thus its build + # cost charged + processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) + + # binary tracker variables for whether a processor is used + # if used, it must abide by capacity constraints, and include build cost if it is a candidate + processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario, logger) + + # tracking unused production + processor_excess_vars = create_processor_excess_output_vars(the_scenario, logger) + + # THIS IS THE OBJECTIVE FUNCTION FOR THE OPTIMIZATION + # ================================================== + + prob = create_opt_problem(logger, the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) + logger.detailed_debug("DEBUG: size of prob: {}".format(sys.getsizeof(prob))) + + prob = create_constraint_unmet_demand(logger, the_scenario, prob, flow_vars, unmet_demand_vars) + logger.detailed_debug("DEBUG: size of prob: {}".format(sys.getsizeof(prob))) + + prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) + + from ftot_pulp import create_constraint_daily_processor_capacity + logger.debug("----- Using create_constraint_daily_processor_capacity method imported from ftot_pulp ------") + prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, + processor_vertex_flow_vars) + + prob = create_primary_processor_vertex_constraints(logger, the_scenario, prob, flow_vars) + + prob = create_constraint_conservation_of_flow_storage_vertices(logger, the_scenario, prob, + flow_vars, processor_excess_vars) + + prob = create_constraint_conservation_of_flow_endcap_nodes(logger, the_scenario, prob, flow_vars, + processor_excess_vars) + + if the_scenario.capacityOn: + logger.info("calling create_constraint_max_route_capacity") + logger.debug('using create_constraint_max_route_capacity method from ftot_pulp') + from ftot_pulp import create_constraint_max_route_capacity + prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) + + logger.info("calling create_constraint_pipeline_capacity") + prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) + + del unmet_demand_vars + + del flow_vars + + # SCENARIO SPECIFIC CONSTRAINTS + + # The problem data is written to an .lp file + prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_c2.lp")) + logger.info("FINISHED: setup PuLP problem for candidate generation") + + return prob + + +# =============================================================================== + + +def solve_pulp_problem(prob_final, the_scenario, logger): + import datetime + + logger.info("START: solve_pulp_problem") + start_time = datetime.datetime.now() + from os import dup, dup2, close + f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') + orig_std_out = dup(1) + dup2(f.fileno(), 1) + + # and relative optimality gap tolerance + status = prob_final.solve(PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance + print('Completion code: %d; Solution status: %s; Best obj value found: %s' % ( + status, LpStatus[prob_final.status], value(prob_final.objective))) + + dup2(orig_std_out, 1) + close(orig_std_out) + f.close() + # The problem is solved using PuLP's choice of Solver + + logger.info("completed calling prob.solve()") + logger.info( + "FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + # THIS IS THE SOLUTION + + # The status of the solution is printed to the screen + # LpStatus key string value numerical value + # LpStatusOptimal ?Optimal? 1 + # LpStatusNotSolved ?Not Solved? 0 + # LpStatusInfeasible ?Infeasible? -1 + # LpStatusUnbounded ?Unbounded? -2 + # LpStatusUndefined ?Undefined? -3 + logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) + + logger.result( + "Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format( + float(value(prob_final.objective)))) + + return prob_final + + +# =============================================================================== + + +def save_pulp_solution(the_scenario, prob, logger): + import datetime + + logger.info("START: save_pulp_solution") + non_zero_variable_count = 0 + + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_cur = db_con.cursor() + # drop the optimal_solution table + # ----------------------------- + db_cur.executescript("drop table if exists optimal_solution;") + + # create the optimal_solution table + # ----------------------------- + db_cur.executescript(""" + create table optimal_solution + ( + variable_name string, + variable_value real + ); + """) + + # insert the optimal data into the DB + # ------------------------------------- + for v in prob.variables(): + if v.varValue > 0.0: + sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format( + v.name, float(v.varValue)) + db_con.execute(sql) + non_zero_variable_count = non_zero_variable_count + 1 + + # query the optimal_solution table in the DB for each variable we care about + # ---------------------------------------------------------------------------- + sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_processors_count = data.fetchone()[0] + logger.info("number of optimal_processors: {}".format(optimal_processors_count)) + + sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_count = data.fetchone()[0] + logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) + sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_sum = data.fetchone()[0] + logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) + logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) + logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format( + optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) + + + sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" + data = db_con.execute(sql) + optimal_edges_count = data.fetchone()[0] + logger.info("number of optimal edges: {}".format(optimal_edges_count)) + logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format( + float(value(prob.objective)) - optimal_unmet_demand_sum * the_scenario.unMetDemandPenalty)) + logger.info( + "Total Scenario Cost = (transportation + unmet demand penalty + " + "processor construction): \t ${0:,.0f}".format( + float(value(prob.objective)))) + start_time = datetime.datetime.now() + logger.info( + "FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + +# =============================================================================== + + +def record_pulp_candidate_gen_solution(the_scenario, logger): + logger.info("START: record_pulp_candidate_gen_solution") + non_zero_variable_count = 0 + + with sqlite3.connect(the_scenario.main_db) as db_con: + + logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) + sql = """ + create table optimal_variables as + select + 'UnmetDemand' as variable_type, + cast(substr(variable_name, 13) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + null as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as edge_count_from_source, + null as miles_travelled + from optimal_solution + where variable_name like 'UnmetDemand%' + union + select + 'Edge' as variable_type, + cast(substr(variable_name, 6) as int) var_id, + variable_value, + edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, + edges.volume*edges.units_conversion_multiplier as converted_volume, + edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, + edges.edge_type, + commodities.commodity_name, + ov.facility_name as o_facility, + dv.facility_name as d_facility, + o_vertex_id, + d_vertex_id, + from_node_id, + to_node_id, + start_day time_period, + edges.commodity_id, + edges.source_facility_id, + s.source_facility_name, + commodities.units, + variable_name, + edges.nx_edge_id, + edges.mode, + edges.mode_oid, + edges.miles, + edges.edge_count_from_source, + edges.miles_travelled + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join commodities on edges.commodity_id = commodities.commodity_ID + left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id + left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id + left outer join source_commodity_ref as s on edges.source_facility_id = s.source_facility_id + where variable_name like 'Edge%' + union + select + 'BuildProcessor' as variable_type, + cast(substr(variable_name, 16) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + 'placeholder' as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as edge_count_from_source, + null as miles_travelled + from optimal_solution + where variable_name like 'Build%'; + """ + db_con.execute("drop table if exists optimal_variables;") + db_con.executescript(sql) + + + sql = """ + create table optimal_variables_c as + select * from optimal_variables + ; + drop table if exists candidate_nodes; + + create table candidate_nodes as + select * + from + + (select pl.minsize, pl.process_id, ov.to_node_id, ov.mode_oid, ov.mode, ov.commodity_id, + sum(variable_value) agg_value, count(variable_value) num_aggregated + from optimal_variables ov, candidate_process_list pl, candidate_process_commodities pc + + where pc.io = 'i' + and pl.process_id = pc.process_id + and ov.commodity_id = pc.commodity_id + and ov.edge_type = 'transport' + group by ov.mode_oid, ov.to_node_id, ov.commodity_id, pl.process_id, pl.minsize, ov.mode) i, + + (select pl.min_aggregation, pl.process_id, ov.from_node_id, ov.commodity_id, + sum(variable_value) agg_value, count(variable_value) num_aggregated, min(edge_count_from_source) as + edge_count, + o_vertex_id + from optimal_variables ov, candidate_process_list pl, candidate_process_commodities pc + where pc.io = 'i' + and pl.process_id = pc.process_id + and ov.commodity_id = pc.commodity_id + and ov.edge_type = 'transport' + group by ov.from_node_id, ov.commodity_id, pl.process_id, pl.min_aggregation, ov.o_vertex_id) o + left outer join networkx_nodes nn + on o.from_node_id = nn.node_id + + where (i.to_node_id = o.from_node_id + and i.process_id = o.process_id + and i.commodity_id = o.commodity_id + and o.agg_value > i.agg_value + and o.agg_value >= o.min_aggregation) + or (o.o_vertex_id is not null and o.agg_value >= o.min_aggregation) + + group by o.min_aggregation, o.process_id, o.from_node_id, o.commodity_id, o.agg_value, o.num_aggregated, + o.o_vertex_id + order by o.agg_value, edge_count + ; + + + """ + db_con.execute("drop table if exists optimal_variables_c;") + db_con.executescript(sql) + + sql_check_candidate_node_count = "select count(*) from candidate_nodes" + db_cur = db_con.execute(sql_check_candidate_node_count) + candidate_node_count = db_cur.fetchone()[0] + if candidate_node_count > 0: + logger.info("number of candidate nodes identified: {}".format(candidate_node_count)) + else: + logger.warning("the candidate nodes table is empty. Hints: " + "(1) increase the maximum raw material transport distance to allow additional flows to " + "aggregate and meet the minimum candidate facility size" + "(2) increase the ammount of material available to flow from raw material producers") + + logger.info("FINISH: record_pulp_candidate_gen_solution") + + +# =============================================================================== + + +def parse_optimal_solution_db(the_scenario, logger): + logger.info("starting parse_optimal_solution") + + optimal_processors = [] + optimal_processor_flows = [] + optimal_route_flows = {} + optimal_unmet_demand = {} + optimal_storage_flows = {} + optimal_excess_material = {} + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # do the Storage Edges + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" + data = db_con.execute(sql) + optimal_storage_edges = data.fetchall() + for edge in optimal_storage_edges: + optimal_storage_flows[edge] = optimal_storage_edges[edge] + + # do the Route Edges + sql = """select + variable_name, variable_value, + cast(substr(variable_name, 6) as int) edge_id, + route_ID, start_day time_period, edges.commodity_id, + o_vertex_id, d_vertex_id, + v1.facility_id o_facility_id, + v2.facility_id d_facility_id + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join vertices v1 on edges.o_vertex_id = v1.vertex_id + join vertices v2 on edges.d_vertex_id = v2.vertex_id + where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; + """ + data = db_con.execute(sql) + optimal_route_edges = data.fetchall() + for edge in optimal_route_edges: + + variable_value = edge[1] + + route_id = edge[3] + + time_period = edge[4] + + commodity_flowed = edge[5] + + od_pair_name = "{}, {}".format(edge[8], edge[9]) + + if route_id not in optimal_route_flows: # first time route_id is used on a day or commodity + optimal_route_flows[route_id] = [[od_pair_name, time_period, commodity_flowed, variable_value]] + + else: # subsequent times route is used on different day or for other commodity + optimal_route_flows[route_id].append([od_pair_name, time_period, commodity_flowed, variable_value]) + + # do the processors + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_candidates_processors = data.fetchall() + for proc in optimal_candidates_processors: + optimal_processors.append(proc) + + # do the processor vertex flows + sql = "select variable_name, variable_value from optimal_solution where variable_name like " \ + "'ProcessorVertexFlow%';" + data = db_con.execute(sql) + optimal_processor_flows_sql = data.fetchall() + for proc in optimal_processor_flows_sql: + optimal_processor_flows.append(proc) + # optimal_biorefs.append(v.name[22:(v.name.find(",")-1)])# find the name from the + + # do the UnmetDemand + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmetdemand = data.fetchall() + for ultimate_destination in optimal_unmetdemand: + v_name = ultimate_destination[0] + v_value = ultimate_destination[1] + + search = re.search('\(.*\)', v_name.replace("'", "")) + + if search: + parts = search.group(0).replace("(", "").replace(")", "").split(",_") + + dest_name = parts[0] + commodity_flowed = parts[2] + if not dest_name in optimal_unmet_demand: + optimal_unmet_demand[dest_name] = {} + + if not commodity_flowed in optimal_unmet_demand[dest_name]: + optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) + else: + optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) + + + logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors + logger.info("length of optimal_processor_flows list: {}".format( + len(optimal_processor_flows))) # a list of optimal processor flows + logger.info("length of optimal_route_flows dict: {}".format( + len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values + logger.info("length of optimal_unmet_demand dict: {}".format( + len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values + + return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material diff --git a/program/ftot_pulp_sourcing.py b/program/ftot_pulp_sourcing.py index 4f41a48..451cee0 100644 --- a/program/ftot_pulp_sourcing.py +++ b/program/ftot_pulp_sourcing.py @@ -1,2129 +1,2130 @@ - -#--------------------------------------------------------------------------------------------------- -# Name: aftot_pulp_sasc -# -# Purpose: PulP optimization - source facility as subcommodity variant -# -#--------------------------------------------------------------------------------------------------- - -import os -import sys - -import ftot_supporting -import ftot_pulp -import sqlite3 -import re -import pdb - -import logging -import datetime - -from ftot import ureg, Q_ - -from collections import defaultdict - -from pulp import * -from ftot_supporting import get_total_runtime_string -#import ftot_supporting_gis -from six import iteritems - -#=================== constants============= -storage = 1 -primary = 0 -fixed_route_max_daily_capacity = 10000000000 -fixed_route_min_daily_capacity = 0 -fixed_schedule_id = 2 -fixed_route_duration = 0 -THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 - - -candidate_processing_facilities = [] - -storage_cost_1 = 0.01 -storage_cost_2 = 0.05 -facility_onsite_storage_max = 10000000000 -facility_onsite_storage_min = 0 -storage_cost_1 = .01 -storage_cost_2 = .05 -fixed_route_max_daily_capacity = 10000000000 -fixed_route_min_daily_capacity = 0 -default_max_capacity = 10000000000 -default_min_capacity = 0 - -#at this level, duplicate edges (i.e. same nx_edge_id, same subcommodity) are only created if they are lower cost. -duplicate_edge_cap = 1 - -#while loop cap - only used for dev to keep runtime reasonable in testing -while_loop_cap = 40 -#debug only -max_transport_distance = 50 - - -def o_sourcing(the_scenario, logger): - import ftot_pulp - pre_setup_pulp_from_optimal(logger, the_scenario) - prob = setup_pulp_problem(the_scenario, logger) - prob = solve_pulp_problem(prob, the_scenario, logger) - save_pulp_solution(the_scenario, prob, logger) - - from ftot_supporting import post_optimization - post_optimization(the_scenario, 'os', logger) - - -#=============================================================================== - - -def delete_all_global_dictionaries(): - global processing_facilities - global processor_storage_vertices - global supply_storage_vertices - global demand_storage_vertices - global processor_vertices - global supply_vertices - global demand_vertices - global storage_facility_vertices - global connector_edges - global storage_edges - global transport_edges - - - processing_facilities = [] - processor_storage_vertices = {} - supply_storage_vertices = {} - demand_storage_vertices = {} - # storage_facility_storage_vertices = {} - processor_vertices = {} - supply_vertices = {} - demand_vertices = {} - storage_facility_vertices = {} - connector_edges = {} - storage_edges = {} - transport_edges = {} - fuel_subtypes = {} - - -def save_existing_solution(logger, the_scenario): - logger.info("START: save_existing_solution") - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - logger.debug("save the current optimal_variables table as optimal_variables_prior") - main_db_con.executescript(""" - drop table if exists optimal_variables_prior; - - create table if not exists optimal_variables_prior as select * from optimal_variables; - - drop table if exists optimal_variables; - """) - - logger.debug("save the current vertices table as vertices_prior and create the new vertices table") - - main_db_con.executescript( - """ - drop table if exists vertices_prior; - - create table if not exists vertices_prior as select * from vertices; - """) - - -#=============================================================================== -def source_as_subcommodity_setup(the_scenario, logger): - logger.info("START: source_as_subcommodity_setup") - multi_commodity_name = "multicommodity" - with sqlite3.connect(the_scenario.main_db) as main_db_con: - main_db_con.executescript(""" - drop table if exists subcommodity - ; - - create table subcommodity(sub_id INTEGER PRIMARY KEY, - source_facility_id integer, - source_facility_name text, - commodity_id integer, - commodity_name text, - units text, - phase_of_matter text, - max_transport_distance text, - CONSTRAINT unique_source_and_name UNIQUE(commodity_name, source_facility_id)) - ; - - - insert into subcommodity ( - source_facility_id, - source_facility_name, - commodity_id, - commodity_name, - units, - phase_of_matter, - max_transport_distance) - - select - f.facility_id, - f.facility_name, - c.commodity_id, - c.commodity_name, - c.units, - c.phase_of_matter, - c.max_transport_distance - - from commodities c, facilities f, facility_type_id ft - where f.facility_type_id = ft.facility_type_id - and ft.facility_type = 'raw_material_producer' - ;""" - ) - main_db_con.execute("""insert or ignore into commodities(commodity_name) values ('{}');""".format(multi_commodity_name)) - main_db_con.execute("""insert or ignore into subcommodity(source_facility_id, source_facility_name, commodity_name, commodity_id) - select sc.source_facility_id, sc.source_facility_name, c.commodity_name, c.commodity_id - from subcommodity sc, commodities c - where c.commodity_name = '{}' - """.format(multi_commodity_name)) - return -#=============================================================================== - -def generate_all_vertices_from_optimal_for_sasc(the_scenario, schedule_dict, schedule_length, logger): - logger.info("START: generate_all_vertices_from_optimal_for_sasc") - #this should be run instead of generate_all_vertices_table, not in addition - - total_potential_production = {} - multi_commodity_name = "multicommodity" - #dictionary items should have the form (vertex, edge_names_in, edge_names_out) - - storage_availability = 1 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - main_db_con.executescript( - """ - drop table if exists vertices; - - create table if not exists vertices ( - vertex_id INTEGER PRIMARY KEY, location_id, - facility_id integer, facility_name text, facility_type_id integer, schedule_day integer, - commodity_id integer, activity_level numeric, storage_vertex binary, - udp numeric, supply numeric, demand numeric, - subcommodity_id integer, - source_facility_id integer, - CONSTRAINT unique_vertex UNIQUE(facility_id, schedule_day, subcommodity_id, storage_vertex)); - - insert or ignore into commodities(commodity_name) values ('{}');""".format(multi_commodity_name)) - - main_db_con.execute("""insert or ignore into subcommodity(source_facility_id, source_facility_name, commodity_name, commodity_id) - select sc.source_facility_id, sc.source_facility_name, c.commodity_name, c.commodity_id - from subcommodity sc, commodities c - where c.commodity_name = '{}' - """.format(multi_commodity_name)) - - #for all facilities that are used in the optimal solution, retrieve facility and facility_commodity information - #each row is a facility_commodity entry then, will get at least 1 vertex; more for days and storage, less for processors? - - #for raw_material_suppliers - #-------------------------------- - - - db_cur = main_db_con.cursor() - db_cur4 = main_db_con.cursor() - counter = 0 - for row in db_cur.execute("select count(distinct facility_id) from facilities;"): - total_facilities = row[0] - - #create vertices for each facility - # facility_type can be "raw_material_producer", "ultimate_destination","processor"; get id from original vertices table for sasc - # use UNION (? check for sqlite) to get a non-repeat list of facilities - - for row_a in db_cur.execute("""select f.facility_id, facility_type, facility_name, location_id, f.facility_type_id, schedule_id - from facilities f, facility_type_id ft, optimal_variables_prior ov - where ignore_facility = '{}' - and f.facility_type_id = ft.facility_type_id - and f.facility_name = ov.o_facility - union - select f.facility_id, facility_type, facility_name, location_id, f.facility_type_id - from facilities f, facility_type_id ft, optimal_variables_prior ov - where ignore_facility = '{}' - and f.facility_type_id = ft.facility_type_id - and f.facility_name = ov.d_facility - ;""".format('false', 'false')): - db_cur2 = main_db_con.cursor() - facility_id = row_a[0] - facility_type = row_a[1] - facility_name = row_a[2] - facility_location_id = row_a[3] - facility_type_id = row_a[4] - schedule_id = row_a[5] - if counter % 10000 == 0: - logger.info("vertices created for {} facilities of {}".format(counter, total_facilities)) - for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): - logger.info('{} vertices created'.format(row_d[0])) - counter = counter + 1 - - - if facility_type == "processor": - - #each processor facility should have 1 input commodity with 1 storage vertex, 1 or more output commodities each with 1 storage vertex, and 1 primary processing vertex - #explode by time, create primary vertex, explode by commodity to create storage vertices; can also create primary vertex for input commodity - #do this for each subcommodity now instead of each commodity - for row_b in db_cur2.execute("""select fc.commodity_id, - ifnull(fc.quantity, 0), - fc.units, - ifnull(c.supertype, c.commodity_name), - fc.io, - mc.commodity_id, - c.commodity_name, - s.sub_id, - s.source_facility_id - from facility_commodities fc, commodities c, commodities mc, subcommodity s - where fc.facility_id = {} - and fc.commodity_id = c.commodity_id - and mc.commodity_name = '{}' - and s.commodity_id = c.commodity_id;""".format(facility_id, multi_commodity_name)): - - commodity_id = row_b[0] - quantity = row_b[1] - units = row_b[2] - commodity_supertype = row_b[3] - io = row_b[4] - id_for_mult_commodities = row_b[5] - commodity_name = row_b[6] - subcommodity_id = row_b[7] - source_facility_id = row_b[8] - #vertices for generic demand type, or any subtype specified by the destination - for day_before, availability in enumerate(schedule_dict[schedule_id]): - if io == 'i': - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, - {},{} - );""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, id_for_mult_commodities, availability, primary, - subcommodity_id, source_facility_id)) - - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {}, {});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, - subcommodity_id, source_facility_id)) - - else: - if commodity_name != 'total_fuel': - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, {}, {});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, subcommodity_id, source_facility_id)) - - elif facility_type == "raw_material_producer": #raw material producer - - # handle as such, exploding by time and commodity - for row_b in db_cur2.execute("""select fc.commodity_id, fc.quantity, fc.units, s.sub_id, s.source_facility_id - from facility_commodities fc, subcommodity s - where fc.facility_id = {} - and s.source_facility_id = {} - and fc.commodity_id = s.commodity_id;""".format(facility_id, facility_id)): - commodity_id = row_b[0] - quantity = row_b[1] - units = row_b[2] - subcommodity_id = row_b[3] - source_facility_id = row_b[4] - - if commodity_id in total_potential_production: - total_potential_production[commodity_id] = total_potential_production[commodity_id] + quantity - else: - total_potential_production[commodity_id] = quantity - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, availability, primary, quantity, - subcommodity_id, source_facility_id)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, - subcommodity_id, source_facility_id)) - - elif facility_type == "storage": #storage facility - # handle as such, exploding by time and commodity - for row_b in db_cur2.execute("""select commodity_id, quantity, units, s.sub_id, s.source_facility_id - from facility_commodities fc, subcommodity s - where fc.facility_id = {} - and fc.commodity_id = s.commodity_id;""".format(facility_id)): - commodity_id = row_b[0] - quantity = row_b[1] - units = row_b[2] - subcommodity_id = row_b[3] - source_facility_id = row_b[4] - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, storage_vertex, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage, - subcommodity_id, source_facility_id)) - - elif facility_type == "ultimate_destination": #ultimate_destination - # handle as such, exploding by time and commodity - for row_b in db_cur2.execute("""select fc.commodity_id, ifnull(fc.quantity, 0), fc.units, - fc.commodity_id, ifnull(c.supertype, c.commodity_name), s.sub_id, s.source_facility_id - from facility_commodities fc, commodities c, subcommodity s - where fc.facility_id = {} - and fc.commodity_id = c.commodity_id - and fc.commodity_id = s.commodity_id;""".format(facility_id)): - commodity_id = row_b[0] - quantity = row_b[1] - units = row_b[2] - commodity_id = row_b[3] - commodity_supertype = row_b[4] - subcommodity_id = row_b[5] - source_facility_id = row_b[6] - - #vertices for generic demand type, or any subtype specified by the destination - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, udp, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, - {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, - commodity_id, availability, primary, quantity, the_scenario.unMetDemandPenalty, - subcommodity_id, source_facility_id)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, - subcommodity_id, source_facility_id)) - - #vertices for other fuel subtypes that match the destination's supertype - #if the subtype is in the commodity table, it is produced by some facility (not ignored) in the scenario - db_cur3 = main_db_con.cursor() - for row_c in db_cur3.execute("""select commodity_id, units from commodities - where supertype = '{}';""".format(commodity_supertype)): - new_commodity_id = row_c[0] - new_units= row_c[1] - for day_before, availability in enumerate(schedule_dict[schedule_id]): - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, udp, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, new_commodity_id, availability, primary, quantity, the_scenario.unMetDemandPenalty, - subcommodity_id, source_facility_id)) - main_db_con.execute("""insert or ignore into vertices ( - location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, - subcommodity_id, source_facility_id) - values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, - {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, new_commodity_id, storage_availability, storage, quantity, - subcommodity_id, source_facility_id)) - - - else: - logger.warning("error, unexpected facility_type: {}, facility_type_id: {}".format(facility_type, facility_type_id)) - - # if it's an origin/supply facility, explode by commodity and time - # if it's a destination/demand facility, explode by commodity and time - # if it's an independent storage facility, explode by commodity and time - # if it's a processing/refining facility, only explode by time - all commodities on the product slate must - # enter and exit the vertex - - #add destination storage vertices for all demand subtypes after all facilities have been iterated, so that we know what fuel subtypes are in this scenario - - - logger.debug("total possible production in scenario: {}".format(total_potential_production)) - -#=============================================================================== - -def add_storage_routes(the_scenario, logger): - # these are loops to and from the same facility; when exploded to edges, they will connect primary to storage vertices, and storage vertices day to day - # is one enough? how many edges will be created per route here? - # will always create edge for this route from storage to storage vertex - # will always create edges for extra routes connecting here - # IF a primary vertex exists, will also create an edge connecting the storage vertex to the primary - - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - logger.debug("create the storage_routes table") - - main_db_con.execute("drop table if exists storage_routes;") - main_db_con.execute("""create table if not exists storage_routes as - select facility_name || '_storage' as route_name, - location_id, - facility_id, - facility_name as o_name, - facility_name as d_name, - {} as cost_1, - {} as cost_2, - 1 as travel_time, - {} as storage_max, - 0 as storage_min - from facilities - where ignore_facility = 'false' - ;""".format(storage_cost_1, storage_cost_2, facility_onsite_storage_max)) - main_db_con.execute("""create table if not exists route_reference( - route_id INTEGER PRIMARY KEY, route_type text, route_name text, scenario_rt_id integer, - CONSTRAINT unique_routes UNIQUE(route_type, route_name, scenario_rt_id));""") - #main_db_con.execute("insert or ignore into route_reference select scenario_rt_id, 'transport', 'see scenario_rt_id', scenario_rt_id from routes;") - main_db_con.execute("insert or ignore into route_reference select null,'storage', route_name, 0 from storage_routes;") - - - logger.debug("storage_routes table created") - - return - -#=============================================================================== -def generate_all_edges_from_optimal_for_sasc(the_scenario, schedule_length, logger): - logger.info("START: generate_all_edges_from_optimal_for_sasc") - #create edges table - #plan to generate start and end days based on nx edge time to traverse and schedule - #can still have route_id, but only for storage routes now; nullable - - multi_commodity_name = "multicommodity" - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - if ('pipeline_crude_trf_rts' in the_scenario.permittedModes) or ('pipeline_prod_trf_rts' in the_scenario.permittedModes) : - logger.info("create indices for the capacity_nodes and pipeline_mapping tables") - main_db_con.executescript( - """ - CREATE INDEX IF NOT EXISTS pm_index ON pipeline_mapping (id, id_field_name, mapping_id_field_name, mapping_id); - CREATE INDEX IF NOT EXISTS cn_index ON capacity_nodes (source, id_field_name, source_OID); - """) - main_db_con.executescript( - """ - drop table if exists edges_prior; - - create table edges_prior as select * from edges; - - drop table if exists edges; - - create table edges (edge_id INTEGER PRIMARY KEY, - route_id integer, - from_node_id integer, - to_node_id integer, - start_day integer, - end_day integer, - commodity_id integer, - o_vertex_id integer, - d_vertex_id integer, - max_edge_capacity numeric, - volume numeric, - capac_minus_volume_zero_floor numeric, - min_edge_capacity numeric, - capacity_units text, - units_conversion_multiplier numeric, - edge_flow_cost numeric, - edge_flow_cost2 numeric, - edge_type text, - nx_edge_id integer, - mode text, - mode_oid integer, - miles numeric, - simple_mode text, - tariff_id numeric, - phase_of_matter text, - subcommodity_id integer, - source_facility_id integer - );""") - - - - db_cur = main_db_con.cursor() - counter = 0 - for row in db_cur.execute("select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): - id_for_mult_commodities = row[0] - for row in db_cur.execute("select count(*) from networkx_edges;"): - total_transport_routes = row[0] - - - - #create transport edges, only between storage vertices and nodes, based on networkx graph and optimal variables - #never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link - #iterate through nx edges: if neither node has a location, create 1 edge per viable commodity - #should also be per day, subject to nx edge schedule - #before creating an edge, check: commodity allowed by nx and max transport distance if not null - #will need nodes per day and commodity? or can I just check that with constraints? - #select data for transport edges - for row_a in db_cur.execute("""select - ne.edge_id, - ifnull(fn.location_id, 'NULL'), - ifnull(tn.location_id, 'NULL'), - ne.mode_source, - ifnull(nec.phase_of_matter_id, 'NULL'), - nec.route_cost, - ne.from_node_id, - ne.to_node_id, - nec.dollar_cost, - ne.miles, - ne.capacity, - ne.artificial, - ne.mode_source_oid - from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, optimal_variables_prior ov - - where ne.from_node_id = fn.node_id - and ne.to_node_id = tn.node_id - and ne.edge_id = nec.edge_id - and ifnull(ne.capacity, 1) > 0 - and ne.edge_id = ov.nx_edge_id - ;"""): - - nx_edge_id = row_a[0] - from_location = row_a[1] - to_location = row_a[2] - mode = row_a[3] - phase_of_matter = row_a[4] - route_cost = row_a[5] - from_node = row_a[6] - to_node = row_a[7] - dollar_cost = row_a[8] - miles = row_a[9] - max_daily_capacity = row_a[10] - mode_oid = row_a[12] - simple_mode = row_a[3].partition('_')[0] - - - db_cur3 = main_db_con.cursor() - - if counter % 10000 == 0: - logger.info("edges created for {} transport links of {}".format(counter, total_transport_routes)) - for row_d in db_cur3.execute("select count(distinct edge_id) from edges;"): - logger.info('{} edges created'.format(row_d[0])) - counter = counter +1 - - tariff_id = 0 - if simple_mode == 'pipeline': - - #find tariff_ids - - sql="select mapping_id from pipeline_mapping where id = {} and id_field_name = 'source_OID' and source = '{}' and mapping_id is not null;".format(mode_oid, mode) - for tariff_row in db_cur3.execute(sql): - tariff_id = tariff_row[0] - - if mode in the_scenario.permittedModes: - - # Edges are placeholders for flow variables - # 4-17: if both ends have no location, iterate through viable commodities and days, create edge - # for all days (restrict by link schedule if called for) - # for all allowed commodities, as currently defined by link phase of matter - - - for day in range(1, schedule_length+1): - if (day+fixed_route_duration <= schedule_length): #if link is traversable in the timeframe - if (simple_mode != 'pipeline' or tariff_id >= 0): - #for allowed commodities - for row_c in db_cur3.execute("select commodity_id, sub_id, source_facility_id from subcommodity where phase_of_matter = '{}' group by commodity_id, sub_id, source_facility_id".format(phase_of_matter)): - db_cur4 = main_db_con.cursor() - commodity_id = row_c[0] - subcommodity_id = row_c[1] - source_facility_id = row_c[2] - #if from_location != 'NULL': - # logger.info("non-null from location nxedge {}, {}, checking for commodity {}".format(from_location,nx_edge_id, commodity_id)) - if(from_location == 'NULL' and to_location == 'NULL'): - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{},{}); - """.format(from_node, to_node, - day, day+fixed_route_duration, commodity_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id,phase_of_matter, subcommodity_id, source_facility_id)) - - elif(from_location != 'NULL' and to_location == 'NULL'): - #for each day and commodity, get the corresponding origin vertex id to include with the edge info - #origin vertex must not be "ultimate_destination - #transport link outgoing from facility - checking fc.io is more thorough than checking if facility type is 'ultimate destination' - #new for bsc, only connect to vertices with matching subcommodity_id (only limited for RMP vertices) - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, subcommodity_id)): - from_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, - min_edge_capacity,edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id,phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{},{}); - """.format(from_node, to_node, - day, day+fixed_route_duration, commodity_id, - from_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) - elif(from_location == 'NULL' and to_location != 'NULL'): - #for each day and commodity, get the corresponding destination vertex id to include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, subcommodity_id)): - to_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, - {}, {}, {}, - '{}',{},'{}',{},{},'{}',{},'{}',{},{}); - """.format(from_node, to_node, - day, day+fixed_route_duration, commodity_id, - to_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) - elif(from_location != 'NULL' and to_location != 'NULL'): - #for each day and commodity, get the corresponding origin and destination vertex ids to include with the edge info - for row_d in db_cur4.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'o'""".format(from_location, day, commodity_id, subcommodity_id)): - from_vertex_id = row_d[0] - db_cur5 = main_db_con.cursor() - for row_e in db_cur5.execute("""select vertex_id - from vertices v, facility_commodities fc - where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} - and v.storage_vertex = 1 - and v.facility_id = fc.facility_id - and v.commodity_id = fc.commodity_id - and fc.io = 'i'""".format(to_location, day, commodity_id, subcommodity_id)): - to_vertex_id = row_d[0] - main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, - start_day, end_day, commodity_id, - o_vertex_id, d_vertex_id, - min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, - {}, {}, {}, - {}, {}, - {}, {}, {}, - '{}',{},'{}', {},{},'{}',{},'{}',{},{}); - """.format(from_node, to_node, - day, day+fixed_route_duration, commodity_id, - from_vertex_id, to_vertex_id, - default_min_capacity, route_cost, dollar_cost, - 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) - - - - - for row_d in db_cur.execute("select count(distinct edge_id) from edges;"): - logger.info('{} edges created'.format(row_d[0])) - - logger.debug("all transport edges created") - - #create storage & connector edges - #for each storage route, get origin storage vertices; if storage vertex exists on day of origin + duration, create a storage edge - for row_a in db_cur.execute("select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, storage_max, storage_min, rr.route_id, location_id, facility_id from storage_routes sr, route_reference rr where sr.route_name = rr.route_name;"): - route_name = row_a[0] - origin_facility_name = row_a[1] - dest_facility_name = row_a[2] - storage_route_cost_1 = row_a[3] - storage_route_cost_2 = row_a[4] - storage_route_travel_time = row_a[5] - max_daily_storage_capacity = row_a[6] - min_daily_storage_capacity = row_a[7] - route_id = row_a[8] - facility_location_id = row_a[9] - facility_id = row_a[10] - - - db_cur2 = main_db_con.cursor() - #storage vertices as origin for this route - #storage routes have no commodity phase restriction so we don't need to check for it here - for row_b in db_cur2.execute("""select v.vertex_id, v.schedule_day, - v.commodity_id, v.storage_vertex, v.subcommodity_id, v.source_facility_id - from vertices v - where v.facility_id = {} - and v.storage_vertex = 1;""".format(facility_id)): - o_vertex_id = row_b[0] - o_schedule_day = row_b[1] - o_commodity_id = row_b[2] - o_storage_vertex = row_b[3] - o_subcommodity_id = row_b[4] - o_source_facility_id = row_b[5] - - - #matching dest vertex for storage - same commodity and facility name, iterate day - db_cur3 = main_db_con.cursor() - for row_c in db_cur3.execute("""select v.vertex_id, v.schedule_day, v.storage_vertex - from vertices v - where v.facility_id = {} - and v.schedule_day = {} - and v.commodity_id = {} - and v.storage_vertex = 1 - and v.vertex_id != {} - and v.subcommodity_id = {};""".format(facility_id, o_schedule_day+storage_route_travel_time, o_commodity_id, o_vertex_id, o_subcommodity_id)): - d_vertex_id = row_c[0] - d_schedule_day = row_c[1] - o_storage_vertex = row_b[2] - main_db_con.execute("""insert or ignore into edges (route_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) - VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( - route_id, o_schedule_day, d_schedule_day, o_commodity_id, o_vertex_id, - d_vertex_id, max_daily_storage_capacity, min_daily_storage_capacity, storage_route_cost_1, 'storage', o_subcommodity_id, o_source_facility_id)) - - #connector edges from storage into primary vertex - processors and destinations - #check commodity direction = i instead of and v.facility_type_id != 'raw_material_producer' - #12-8 upates to account for multi-commodity processor vertices - #subcommodity has to match even if multi commodity - keep track of different processor vertex copies - db_cur3 = main_db_con.cursor() - for row_c in db_cur3.execute("""select d.vertex_id, d.schedule_day - from vertices d, facility_commodities fc - where d.facility_id = {} - and d.schedule_day = {} - and (d.commodity_id = {} or d.commodity_id = {}) - and d.storage_vertex = 0 - and d.facility_id = fc.facility_id - and fc.commodity_id = {} - and fc.io = 'i' - and d.source_facility_id = {} - ; - """.format(facility_id, o_schedule_day, o_commodity_id, id_for_mult_commodities, o_commodity_id,o_source_facility_id)): - d_vertex_id = row_c[0] - d_schedule_day = row_c[1] - connector_cost = 0 - main_db_con.execute("""insert or ignore into edges (route_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) - VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( - route_id, o_schedule_day, d_schedule_day, o_commodity_id, o_vertex_id, - d_vertex_id, default_max_capacity, default_min_capacity, connector_cost, 'connector', o_subcommodity_id, o_source_facility_id)) - - - #create remaining connector edges - #primary to storage vertex - supplier, destination, processor; not needed for storage facilities - #same day, same commodity (unless processor), no cost; purpose is to separate control of flows - #into and out of the system from flows within the system (transport & storage) - #and v.facility_type_id != 'ultimate_destination' - #create connectors ending at storage - - - d_vertex_id = row_b[0] - d_schedule_day = row_b[1] - d_commodity_id = row_b[2] - d_storage_vertex = row_b[3] - d_subcommodity_id = row_b[4] - d_source_facility_id = row_b[5] - - - db_cur3 = main_db_con.cursor() - for row_c in db_cur3.execute("""select vertex_id, schedule_day from vertices v, facility_commodities fc - where v.facility_id = {} - and v.schedule_day = {} - and (v.commodity_id = {} or v.commodity_id = {}) - and v.storage_vertex = 0 - and v.facility_id = fc.facility_id - and fc.commodity_id = {} - and fc.io = 'o' - and v.source_facility_id = {} - ; - """.format(facility_id, d_schedule_day, d_commodity_id, id_for_mult_commodities, d_commodity_id,d_source_facility_id)): - o_vertex_id = row_c[0] - o_schedule_day = row_c[1] - connector_cost = 0 - main_db_con.execute("""insert or ignore into edges (route_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) - VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( - route_id, o_schedule_day, d_schedule_day, d_commodity_id, - o_vertex_id, d_vertex_id, default_max_capacity, default_min_capacity, - connector_cost, 'connector', d_subcommodity_id, d_source_facility_id)) - - logger.info("all edges created") - logger.info("create an index for the edges table by nodes") - - sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( - edge_id, route_id, from_node_id, to_node_id, commodity_id, - start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, - max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, - edge_type, nx_edge_id, mode, mode_oid, miles, subcommodity_id, source_facility_id);""") - db_cur.execute(sql) - - return - -#=============================================================================== -def set_edge_capacity_and_volume(the_scenario, logger): - - logger.info("START: set_edge_capacity_and_volume") - - multi_commodity_name = "multicommodity" - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - - logger.debug("starting to record volume and capacity for non-pipeline edges") - - main_db_con.execute("update edges set volume = (select ifnull(ne.volume,0) from networkx_edges ne where ne.edge_id = edges.nx_edge_id ) where simple_mode in ('rail','road','water');") - main_db_con.execute("update edges set max_edge_capacity = (select ne.capacity from networkx_edges ne where ne.edge_id = edges.nx_edge_id) where simple_mode in ('rail','road','water');") - logger.debug("volume and capacity recorded for non-pipeline edges") - - logger.debug("starting to record volume and capacity for pipeline edges") -## - main_db_con.executescript("""update edges set volume = - (select l.background_flow - from pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where edges.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(edges.mode, l.source)>0) - where simple_mode = 'pipeline' - ; - - update edges set max_edge_capacity = - (select l.capac - from pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where edges.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(edges.mode, l.source)>0) - where simple_mode = 'pipeline' - ;""") - - logger.debug("volume and capacity recorded for pipeline edges") - logger.debug("starting to record units and conversion multiplier") - logger.debug("update edges set capacity_units") - - main_db_con.execute("""update edges - set capacity_units = - (case when simple_mode = 'pipeline' then 'kbarrels' - when simple_mode = 'road' then 'truckload' - when simple_mode = 'rail' then 'railcar' - when simple_mode = 'water' then 'barge' - else 'unexpected mode' end) - ;""") - - logger.debug("update edges set units_conversion_multiplier;") - - main_db_con.execute("""update edges - set units_conversion_multiplier = - (case when simple_mode = 'pipeline' and phase_of_matter = 'liquid' then {} - when simple_mode = 'road' and phase_of_matter = 'liquid' then {} - when simple_mode = 'road' and phase_of_matter = 'solid' then {} - when simple_mode = 'rail' and phase_of_matter = 'liquid' then {} - when simple_mode = 'rail' and phase_of_matter = 'solid' then {} - when simple_mode = 'water' and phase_of_matter = 'liquid' then {} - when simple_mode = 'water' and phase_of_matter = 'solid' then {} - else 1 end) - ;""".format(THOUSAND_GALLONS_PER_THOUSAND_BARRELS, - the_scenario.truck_load_liquid.magnitude, - the_scenario.truck_load_solid.magnitude, - the_scenario.railcar_load_liquid.magnitude, - the_scenario.railcar_load_solid.magnitude, - the_scenario.barge_load_liquid.magnitude, - the_scenario.barge_load_solid.magnitude, - )) - - logger.debug("units and conversion multiplier recorded for all edges; starting capacity minus volume") - - main_db_con.execute("""update edges - set capac_minus_volume_zero_floor = - max((select (max_edge_capacity - ifnull(volume,0)) where max_edge_capacity is not null),0) - where max_edge_capacity is not null - ;""") - - logger.debug("capacity minus volume (minimum set to zero) recorded for all edges") - - return - - -#=============================================================================== -def pre_setup_pulp_from_optimal(logger, the_scenario): - - logger.info("START: pre_setup_pulp") - - source_as_subcommodity_setup(the_scenario, logger) - - schedule_dict, schedule_length = ftot_pulp.generate_schedules(the_scenario, logger) - - generate_all_vertices_from_optimal_for_sasc(the_scenario, schedule_dict, schedule_length, logger) - - add_storage_routes(the_scenario, logger) - generate_all_edges_from_optimal_for_sasc(the_scenario, schedule_length, logger) - set_edge_capacity_and_volume(the_scenario, logger) - - logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) - - - return - -#=============================================================================== - -def create_flow_vars(the_scenario, logger): - # all_edges is a list of strings to be the keys for LPVariable.dict - - # we have a table called edges. - # call helper method to get list of unique IDs from the Edges table. - # use a the rowid as a simple unique integer index - edge_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, subcommodity_id - from edges;""") - edge_list_data = edge_list_cur.fetchall() - counter = 0 - for row in edge_list_data: - if counter % 500000 == 0: - logger.info("processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) - counter += 1 - #create an edge for each commodity allowed on this link - this construction may change as specific commodity restrictions are added - #running just with nodes for now, will add proper facility info and storage back soon - edge_list.append((row[0])) - - logger.debug("MNP DEBUG: start assign flow_var with edge_list") - - flow_var = LpVariable.dicts("Edge",edge_list,0,None) - logger.debug("MNP DEBUG: Size of flow_var: {:,.0f}".format(sys.getsizeof(flow_var))) - #delete edge_list_data - edge_list_data = [] - return flow_var - # flow_var is the flow on each arc, being determined; this can be defined any time after all_arcs is defined - -#=============================================================================== -def create_unmet_demand_vars(the_scenario, logger): - - demand_var_list = [] - #may create vertices with zero demand, but only for commodities that the facility has demand for at some point - - - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) top_level_commodity_name, v.udp - from vertices v, commodities c, facility_type_id ft, facilities f - where v.commodity_id = c.commodity_id - and ft.facility_type = "ultimate_destination" - and v.storage_vertex = 0 - and v.facility_type_id = ft.facility_type_id - and v.facility_id = f.facility_id - and f.ignore_facility = 'false' - group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) - ;""".format('')): - #facility_id, day, and simplified commodity name - demand_var_list.append((row[0], row[1], row[2], row[3])) - - unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) - - return unmet_demand_var - -#=============================================================================== - -def create_candidate_processor_build_vars(the_scenario, logger): - processors_build_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("select f.facility_id from facilities f, facility_type_id ft where f.facility_type_id = ft.facility_type_id and facility_type = 'processor' and candidate = 1 and ignore_facility = 'false' group by facility_id;"): - #grab all candidate processor facility IDs - processors_build_list.append(row[0]) - - processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list,0,None, 'Binary') - - - - return processor_build_var -#=============================================================================== -def create_binary_processor_vertex_flow_vars(the_scenario): - processors_flow_var_list = [] - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row in db_cur.execute("""select v.facility_id, v.schedule_day - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 0 - group by v.facility_id, v.schedule_day;"""): - #facility_id, day - processors_flow_var_list.append((row[0], row[1])) - - processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0,None, 'Binary') - - - return processor_flow_var -#=============================================================================== -def create_processor_excess_output_vars(the_scenario): - excess_var_list = [] - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - xs_cur = db_cur.execute(""" - select vertex_id, commodity_id - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and facility_type = 'processor' - and storage_vertex = 1;""") - #facility_id, day, and simplified commodity name - xs_data = xs_cur.fetchall() - for row in xs_data: - excess_var_list.append(row[0]) - - - excess_var = LpVariable.dicts("XS", excess_var_list, 0,None) - - - return excess_var -#=============================================================================== - -def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): - - logger.debug("START: create_constraint_unmet_demand") - - - - #apply activity_level to get corresponding actual demand for var - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - # var has form(facility_name, day, simple_fuel) - #unmet demand commodity should be simple_fuel = supertype - logger.debug("MNP: DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_var))) - - demand_met_dict = defaultdict(list) - actual_demand_dict = {} - - demand_met = [] - # want to specify that all edges leading into this vertex + unmet demand = total demand - #demand primary (non-storage) vertices - - db_cur = main_db_con.cursor() - #each row_a is a primary vertex whose edges in contributes to the met demand of var - #will have one row for each fuel subtype in the scenario - unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.subcommodity_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - from vertices v, commodities c, subcommodity sc, facility_type_id ft, facilities f, edges e - where v.facility_id = f.facility_id - and ft.facility_type = 'ultimate_destination' - and f.facility_type_id = ft.facility_type_id - and f.ignore_facility = 'false' - and v.facility_type_id = ft.facility_type_id - and v.storage_vertex = 0 - and c.commodity_id = v.commodity_id - and sc.sub_id = v.subcommodity_id - and e.d_vertex_id = v.vertex_id - group by v.vertex_id, v.commodity_id, - v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.subcommodity_id, - v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id - ;""") - - unmet_data = unmet_data.fetchall() - for row_a in unmet_data: - - primary_vertex_id = row_a[0] - commodity_id = row_a[1] - var_full_demand = row_a[2] - proportion_of_supertype = row_a[3] - var_activity_level = row_a[4] - subc_id = row_a[5] - facility_id = row_a[6] - day = row_a[7] - top_level_commodity = row_a[8] - udp = row_a[9] - edge_id = row_a[10] - var_actual_demand = var_full_demand*var_activity_level - - #next get inbound edges and apply appropriate modifier proportion to get how much of var's demand they satisfy - demand_met_dict[(facility_id, day, top_level_commodity, udp)].append(flow_var[edge_id]*proportion_of_supertype) - actual_demand_dict[(facility_id, day, top_level_commodity, udp)]=var_actual_demand - - for var in unmet_demand_var: - if var in demand_met_dict: - #then there are some edges in - prob += lpSum(demand_met_dict[var]) == actual_demand_dict[var] - unmet_demand_var[var], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(var[0], var[1], var[2]) - else: - #no edges in, so unmet demand equals full demand - prob += actual_demand_dict[var] == unmet_demand_var[var], "constraint set unmet demand variable for facility {}, day {}, commodity {} - no edges able to meet demand".format(var[0], var[1], var[2]) - - - - logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") - return prob - -#=============================================================================== -# create_constraint_max_flow_out_of_supply_vertex -# primary vertices only -# flow out of a vertex <= supply of the vertex, true for every day and commodity -def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - - #for each primary (non-storage) supply vertex - #Assumption - each RMP produces a single commodity - - #Assumption - only one vertex exists per day per RMP (no multi commodity or subcommodity) - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - for row_a in db_cur.execute("""select vertex_id, activity_level, supply - from vertices v, facility_type_id ft - where v.facility_type_id = ft.facility_type_id - and ft.facility_type = 'raw_material_producer' - and storage_vertex = 0;"""): - supply_vertex_id = row_a[0] - activity_level = row_a[1] - max_daily_supply = row_a[2] - actual_vertex_supply = activity_level*max_daily_supply - - flow_out = [] - db_cur2 = main_db_con.cursor() - #select all edges leaving that vertex and sum their flows - #should be a single connector edge - for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): - edge_id = row_b[0] - flow_out.append(flow_var[edge_id]) - - prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format(actual_vertex_supply, supply_vertex_id) - #could easily add human-readable vertex info to this if desirable - - logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") - return prob - #force flow out of origins to be <= supply - -#=============================================================================== - - -# for all of these vertices, flow in always == flow out -def create_processor_constraints(logger, the_scenario, prob, flow_var, processor_excess_vars, processor_build_vars, processor_daily_flow_vars): - logger.debug("STARTING: create_processor_constraints - capacity and conservation of flow") - node_counter = 0 - node_constraint_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - - logger.debug("conservation of flow, primary processor vertices:") - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - ifnull(f.candidate, 0) candidate_check, - e.subcommodity_id, - e.source_facility_id, - v.subcommodity_id, - v.source_facility_id - from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f - join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) - where ft.facility_type = 'processor' - and v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 0 - and v.facility_id = fc.facility_id - and fc.commodity_id = c.commodity_id - and fc.commodity_id = e.commodity_id - group by v.vertex_id, - in_or_out_edge, - constraint_day, - e.commodity_id, - e.mode, - e.edge_id, - nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, - fc.io, - v.activity_level, - candidate_check, - e.subcommodity_id, - e.source_facility_id, - v.subcommodity_id, - v.source_facility_id - order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id - ;""" - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - sql_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info("execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - sql_data = sql_data.fetchall() - logger.info("fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - - #keys are processor vertices - flow_in = {} - flow_out_lists = {} - - - for row_a in sql_data: - - vertex_id = row_a[0] - in_or_out_edge = row_a[1] - constraint_day = row_a[2] - commodity_id = row_a[3] - mode = row_a[4] - edge_id = row_a[5] - nx_edge_id = row_a[6] - quantity = float(row_a[7]) - facility_id = row_a[8] - commodity_name = row_a[9] - fc_io_commodity = row_a[10] - activity_level = row_a[11] - is_candidate = row_a[12] - edge_subcommodity_id = row_a[13] - edge_source_facility_id = row_a[14] - vertex_subcommodity_id = row_a[15] - vertex_source_facility_id = row_a[16] - - #edges should only connect to vertices with matching source facility - #if that's not true there could be issues here - - - if in_or_out_edge == 'in': - flow_in[vertex_id] = (flow_var[edge_id], quantity, commodity_name, activity_level, is_candidate, facility_id, vertex_source_facility_id) - elif in_or_out_edge == 'out': - flow_out_lists[(vertex_id, commodity_id)] = (flow_var[edge_id], quantity, commodity_name, facility_id, vertex_source_facility_id) - - - #require ratio to match for each output commodityalso do processor capacity constraint here - - for key, value in iteritems(flow_out_lists): - flow_out = value[0] - out_quantity = value[1] - vertex_id = key[0] - out_commodity = value[2] - flow_in_var = flow_in[vertex_id][0] - flow_in_quant = flow_in[vertex_id][1] - flow_in_commodity = flow_in[vertex_id][2] - required_flow_out = flow_out/out_quantity - required_flow_in = flow_in_var/flow_in_quant - - - # Sort edges by commodity and whether they're going in or out of the node - - prob += required_flow_out == required_flow_in, "conservation of flow, processor vertex {}, processor facility {}, commodity {} to {}, source facility {}".format(vertex_id, facility_id, flow_in_commodity, out_commodity, vertex_source_facility_id) - - logger.debug("FINISHED: processor conservation of flow and ratio constraint") - - - logger.debug("FINISHED: create_processor_constraints") - return prob - -def create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_var, processor_excess_vars): - logger.debug("STARTING: create_constraint_conservation_of_flow") - node_counter = 0 - node_constraint_counter = 0 - storage_vertex_constraint_counter = 0 - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - - logger.info("conservation of flow, storage vertices:") - #storage vertices, any facility type - #these have at most one direction of transport edges, so no need to track mode - sql = """select v.vertex_id, - (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, - (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id, v.facility_id, c.commodity_name, - v.activity_level, - ft.facility_type - - from vertices v, facility_type_id ft, commodities c, facilities f - join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) and (e.o_vertex_id = v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) - - where v.facility_id = f.facility_id - and ft.facility_type_id = v.facility_type_id - and storage_vertex = 1 - and v.commodity_id = c.commodity_id - - group by v.vertex_id, - in_or_out_edge, - constraint_day, - v.commodity_id, - e.edge_id, - nx_edge_id,v.facility_id, c.commodity_name, - v.activity_level - - order by v.facility_id, v.vertex_id, e.edge_id - ;""" - - # get the data from sql and see how long it takes. - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - vertexid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - vertexid_data = vertexid_data.fetchall() - logger.info("fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - - flow_in_lists = {} - flow_out_lists = {} - for row_v in vertexid_data: - vertex_id = row_v[0] - in_or_out_edge = row_v[1] - constraint_day = row_v[2] - commodity_id = row_v[3] - edge_id = row_v[4] - nx_edge_id = row_v[5] - facility_id = row_v[6] - commodity_name = row_v[7] - activity_level = row_v[8] - facility_type = row_v[9] - - if in_or_out_edge == 'in': - flow_in_lists.setdefault((vertex_id, commodity_name, constraint_day, facility_type), []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((vertex_id, commodity_name, constraint_day, facility_type), []).append(flow_var[edge_id]) - - logger.info("adding processor excess variabless to conservation of flow") - for key, value in iteritems(flow_out_lists): - vertex_id = key[0] - commodity_name = key[1] - day = key[2] - facility_type = key[3] - if facility_type == 'processor': - flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) - - - for key, value in iteritems(flow_out_lists): - - if key in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - else: - prob += lpSum(flow_out_lists[key]) == lpSum(0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - for key, value in iteritems(flow_in_lists): - - if key not in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum(0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) - storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 - - logger.info("total conservation of flow constraints created on nodes: {}".format(storage_vertex_constraint_counter)) - - - - logger.info("conservation of flow, nx_nodes:") - #non-vertex nodes, no facility type, connect 2 transport edges over matching days and commodities - #for each day, get all edges in and out of the node. Sort them by commodity and whether they're going in or out of the node - sql = """select nn.node_id, - (case when e.from_node_id = nn.node_id then 'out' when e.to_node_id = nn.node_id then 'in' else 'error' end) in_or_out_edge, - (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 end) constraint_day, - e.commodity_id, - ifnull(mode, 'NULL'), - e.edge_id, nx_edge_id, - miles, - (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, - e.subcommodity_id - from networkx_nodes nn - join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) - where nn.location_id is null - order by node_id, commodity_id, - (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 end), - in_or_out_edge, e.subcommodity_id - ;""" - - # get the data from sql and see how long it takes. - logger.info("Starting the long step:") - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - nodeid_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for :") - logger.info("execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - nodeid_data = nodeid_data.fetchall() - logger.info("fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - - flow_in_lists = {} - flow_out_lists = {} - - for row_a in nodeid_data: - node_id = row_a[0] - in_or_out_edge = row_a[1] - constraint_day = row_a[2] - commodity_id = row_a[3] - mode = row_a[4] - edge_id = row_a[5] - nx_edge_id = row_a[6] - miles = row_a[7] - intermodal = row_a[8] - subcommodity_id = row_a[9] - - #if node is not intermodal, conservation of flow holds per mode; - #if intermodal, then across modes - if intermodal == 'N': - if in_or_out_edge == 'in': - flow_in_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day, mode), []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day, mode), []).append(flow_var[edge_id]) - else: - if in_or_out_edge == 'in': - flow_in_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day), []).append(flow_var[edge_id]) - elif in_or_out_edge == 'out': - flow_out_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day), []).append(flow_var[edge_id]) - - - for key, value in iteritems(flow_out_lists): - node_id = key[0] - intermodal_flag = key[1] - subcommodity_id = key[2] - day = key[3] - if len(key) == 5: - node_mode = key[4] - else: node_mode = 'intermodal' - if key in flow_in_lists: - prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[key]), "conservation of flow, nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - else: - prob += lpSum(flow_out_lists[key]) == lpSum(0), "conservation of flow (zero out), nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - for key, value in iteritems(flow_in_lists): - node_id = key[0] - intermodal_flag = key[1] - subcommodity_id = key[2] - day = key[3] - if len(key) == 5: - node_mode = key[4] - else: node_mode = 'intermodal' - - if key not in flow_out_lists: - prob += lpSum(flow_in_lists[key]) == lpSum(0), "conservation of flow (zero in), nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) - node_constraint_counter = node_constraint_counter + 1 - - logger.info("total conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) - - #Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints - - logger.debug("FINISHED: create_constraint_conservation_of_flow") - - return prob - -# set route capacity by phase of matter - may connect multiple facilities, definitely multiple edges -#do storage capacity separately, even though it is also routes -def create_constraint_max_route_capacity(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_max_route_capacity") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) - #min_capacity_level must be a number from 0 to 1, inclusive - #min_capacity_level is only relevant when background flows are turned on - #it sets a floor to how much capacity can be reduced by volume. - #min_capacity_level = .25 means route capacity will never be less than 25% of full capacity, even if "volume" would otherwise restrict it further - #min_capacity_level = 0 allows a route to be made unavailable for FTOT flow if base volume is too high - #this currently applies to all modes; could be made mode specific - logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - #capacity for storage routes - sql = """select - rr.route_id, sr.storage_max, sr.route_name, e.edge_id, e.start_day - from route_reference rr - join storage_routes sr on sr.route_name = rr.route_name - join edges e on rr.route_id = e.route_id - ;""" - # get the data from sql and see how long it takes. - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - storage_edge_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for storage edges:") - logger.info("execute for edges for storage - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - storage_edge_data = storage_edge_data.fetchall() - logger.info("fetchall edges for storage - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in storage_edge_data: - route_id = row_a[0] - aggregate_storage_capac = row_a[1] - storage_route_name = row_a[2] - edge_id = row_a[3] - start_day = row_a[4] - - flow_lists.setdefault((route_id, aggregate_storage_capac, storage_route_name, start_day), []).append(flow_var[edge_id]) - - - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on storage route {} named {} for day {}".format(key[0], key[2], key[3]) - - logger.debug("route_capacity constraints created for all storage routes") - - - - #capacity for transport routes - #Assumption - all flowing material is in kgal, all flow is summed on a single non-pipeline nx edge - sql = """select e.edge_id, e.nx_edge_id, e.max_edge_capacity, e.start_day, e.simple_mode, e.phase_of_matter, e.capac_minus_volume_zero_floor - from edges e - where e.max_edge_capacity is not null - and e.simple_mode != 'pipeline' - ;""" - # get the data from sql and see how long it takes. - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - route_capac_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for transport edges:") - logger.info("execute for non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - route_capac_data = route_capac_data.fetchall() - logger.info("fetchall non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in route_capac_data: - edge_id = row_a[0] - nx_edge_id = row_a[1] - nx_edge_capacity = row_a[2] - start_day = row_a[3] - simple_mode = row_a[4] - phase_of_matter = row_a[5] - capac_minus_background_flow = max(row_a[6], 0) - min_restricted_capacity = max(capac_minus_background_flow, nx_edge_capacity*the_scenario.minCapacityLevel) - - if simple_mode in the_scenario.backgroundFlowModes: - use_capacity = min_restricted_capacity - else: use_capacity = nx_edge_capacity - - #flow is in thousand gallons (kgal), for liquid, or metric tons, for solid - #capacity is in truckload, rail car, barge, or pipeline movement per day - # if mode is road and phase is liquid, capacity is in truckloads per day, we want it in kgal - # ftot_supporting_gis tells us that there are 8 kgal per truckload, so capacity * 8 gives us correct units or kgal per day - # use capacity * ftot_supporting_gis multiplier to get capacity in correct flow units - - multiplier = 1 #unless otherwise specified - if simple_mode == 'road': - if phase_of_matter == 'liquid': - multiplier = the_scenario.truck_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.truck_load_solid.magnitude - elif simple_mode == 'water': - if phase_of_matter == 'liquid': - multiplier = the_scenario.barge_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.barge_load_solid.magnitude - elif simple_mode == 'rail': - if phase_of_matter == 'liquid': - multiplier = the_scenario.railcar_load_liquid.magnitude - elif phase_of_matter == 'solid': - multiplier = the_scenario.railcar_load_solid.magnitude - - converted_capacity = use_capacity*multiplier - - flow_lists.setdefault((nx_edge_id, converted_capacity, start_day), []).append(flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on nx edge {} for day {}".format(key[0], key[2]) - - logger.debug("route_capacity constraints created for all non-pipeline transport routes") - - - logger.debug("FINISHED: create_constraint_max_route_capacity") - return prob - -def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): - logger.debug("STARTING: create_constraint_pipeline_capacity") - logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) - logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) - logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) - - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - db_cur = main_db_con.cursor() - - #capacity for pipeline tariff routes - #with sasc, may have multiple flows per segment, slightly diff commodities - sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow allowed_flow, l.source, e.mode, instr(e.mode, l.source) - from edges e, pipeline_mapping pm, - (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, - max(cn.volume) background_flow, source - from capacity_nodes cn - where cn.id_field_name = 'MASTER_OID' - and ifnull(cn.capacity,0)>0 - group by link_id) l - - where e.tariff_id = pm.id - and pm.id_field_name = 'tariff_ID' - and pm.mapping_id_field_name = 'MASTER_OID' - and l.id_field_name = 'MASTER_OID' - and pm.mapping_id = l.link_id - and instr(e.mode, l.source)>0 - group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source - ;""" - # capacity needs to be shared over link_id for any edge_id associated with that link - - logger.info("Starting the execute") - execute_start_time = datetime.datetime.now() - pipeline_capac_data = db_cur.execute(sql) - logger.info("Done with the execute fetch all for transport edges:") - logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) - - logger.info("Starting the fetchall") - fetchall_start_time = datetime.datetime.now() - pipeline_capac_data = pipeline_capac_data.fetchall() - logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) - - flow_lists = {} - - for row_a in pipeline_capac_data: - edge_id = row_a[0] - tariff_id = row_a[1] - link_id = row_a[2] - # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall - # Link capacity * 42 is now in kgal per day, to match flow in kgal - link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS*row_a[3] - start_day = row_a[4] - capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS*row_a[5],0) - min_restricted_capacity = max(capac_minus_background_flow_kgal, link_capacity_kgal_per_day*the_scenario.minCapacityLevel) - - capacity_nodes_mode_source = row_a[6] - edge_mode = row_a[7] - mode_match_check = row_a[8] - if 'pipeline' in the_scenario.backgroundFlowModes: - link_use_capacity = min_restricted_capacity - else: link_use_capacity = link_capacity_kgal_per_day - - #add flow from all relevant edges, for one start; may be multiple tariffs - flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) - - for key, flow in iteritems(flow_lists): - prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format(key[0], key[3], key[2]) - - logger.debug("pipeline capacity constraints created for all transport routes") - - - logger.debug("FINISHED: create_constraint_pipeline_capacity") - return prob - -#=============================================================================== -## - -def setup_pulp_problem(the_scenario, logger): - - - logger.info("START: setup PuLP problem") - - # flow_var is the flow on each edge by commodity and day. - # the optimal value of flow_var will be solved by PuLP - logger.info("calling create_flow_vars") - flow_vars = create_flow_vars(the_scenario, logger) - - logger.info("calling create_unmet_demand_vars") - #unmet_demand_var is the unmet demand at each destination, being determined - unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) - - logger.info("calling create_candidate_processor_build_vars") - #processor_build_vars is the binary variable indicating whether a candidate processor is used and thus its build cost charged - processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) - - logger.info("calling create_binary_processor_vertex_flow_vars") - #binary tracker variables - processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario) - - logger.info("calling create_processor_excess_output_vars") - #tracking unused production - processor_excess_vars = create_processor_excess_output_vars(the_scenario) - - - - # THIS IS THE OBJECTIVE FUCTION FOR THE OPTIMIZATION - # ================================================== - logger.info("calling create_opt_problem") - - prob = ftot_pulp.create_opt_problem(logger,the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) - logger.info("MNP DEBUG: size of prob: {}".format(sys.getsizeof(prob))) - - logger.info("calling create_constraint_unmet_demand") - prob = create_constraint_unmet_demand(logger,the_scenario, prob, flow_vars, unmet_demand_vars) - logger.debug("MNP DEBUG: size of prob: {}".format(sys.getsizeof(prob))) - - logger.info("calling create_constraint_max_flow_out_of_supply_vertex") - prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) - - - logger.info("calling create_constraint_daily_processor_capacity from ftot_pulp") - from ftot_pulp import create_constraint_daily_processor_capacity - prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, processor_vertex_flow_vars) - - logger.info("calling create_processor_constraints") - prob = create_processor_constraints(logger, the_scenario, prob, flow_vars, processor_excess_vars, processor_build_vars, processor_vertex_flow_vars) - - logger.info("calling create_constraint_conservation_of_flow") - prob = create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_vars, processor_excess_vars) - - if the_scenario.capacityOn: - - logger.info("calling create_constraint_max_route_capacity") - prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) - - logger.info("calling create_constraint_pipeline_capacity") - prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) - - del(unmet_demand_vars) - - del(flow_vars) - - # SCENARIO SPECIFIC CONSTRAINTS - - logger.info("calling write LP file") - prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_sasc.lp")) - logger.info("FINISHED: setup PuLP problem") - - return prob - -###=============================================================================== -def solve_pulp_problem(prob_final, the_scenario, logger): - - import datetime - - logger.info("START: prob.solve()") - #begin new prob.solve logging, 9-04-2018 - start_time = datetime.datetime.now() - from os import dup, dup2, close - f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') - orig_std_out = dup(1) - dup2(f.fileno(), 1) - - #status = prob_final.solve (PULP_CBC_CMD(maxSeconds = i_max_sec, fracGap = d_opt_gap, msg=1)) # CBC time limit and relative optimality gap tolerance - status = prob_final.solve (PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance - print('Completion code: %d; Solution status: %s; Best obj value found: %s' % (status, LpStatus[prob_final.status], value(prob_final.objective))) - - dup2(orig_std_out, 1) - close(orig_std_out) - f.close() - #end new prob.solve logging, 9-04-2018 - - logger.info("completed calling prob.solve()") - logger.info("FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - # THIS IS THE SOLUTION - # The status of the solution is printed to the screen - ##LpStatus key string value numerical value - ##LpStatusOptimal ?Optimal? 1 - ##LpStatusNotSolved ?Not Solved? 0 - ##LpStatusInfeasible ?Infeasible? -1 - ##LpStatusUnbounded ?Unbounded? -2 - ##LpStatusUndefined ?Undefined? -3 - logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) - - logger.result("Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format(float(value(prob_final.objective)))) - - return prob_final -#------------------------------------------------------------------------------ -def save_pulp_solution(the_scenario, prob, logger): - import datetime - - logger.info("START: save_pulp_solution") - non_zero_variable_count = 0 - - - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_cur = db_con.cursor() - # drop the optimal_solution table - #----------------------------- - db_cur.executescript("drop table if exists optimal_solution;") - - # create the optimal_solution table - #----------------------------- - db_cur.executescript(""" - create table optimal_solution - ( - variable_name string, - variable_value real - ); - """) - - - # insert the optimal data into the DB - #------------------------------------- - for v in prob.variables(): - if v.varValue > 0.0: - - sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format(v.name, float(v.varValue)) - db_con.execute(sql) - non_zero_variable_count = non_zero_variable_count + 1 - - logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) - - sql = """ - create table optimal_variables as - select - 'UnmetDemand' as variable_type, - cast(substr(variable_name, 13) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - null as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as subcommodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as original_facility, - null as final_facility, - null as prior_edge - from optimal_solution - where variable_name like 'UnmetDemand%' - union - select - 'Edge' as variable_type, - cast(substr(variable_name, 6) as int) var_id, - variable_value, - edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, - edges.volume*edges.units_conversion_multiplier as converted_volume, - edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, - edges.edge_type, - commodities.commodity_name, - ov.facility_name as o_facility, - dv.facility_name as d_facility, - o_vertex_id, - d_vertex_id, - from_node_id, - to_node_id, - start_day time_period, - edges.commodity_id, - edges.subcommodity_id, - edges.source_facility_id, - s.source_facility_name, - commodities.units, - variable_name, - edges.nx_edge_id, - edges.mode, - edges.mode_oid, - edges.miles, - null as original_facility, - null as final_facility, - null as prior_edge - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join commodities on edges.commodity_id = commodities.commodity_ID - left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id - left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id - left outer join subcommodity as s on edges.subcommodity_id = s.sub_id - where variable_name like 'Edge%' - union - select - 'BuildProcessor' as variable_type, - cast(substr(variable_name, 16) as int) var_id, - variable_value, - null as converted_capacity, - null as converted_volume, - null as converted_capac_minus_volume, - null as edge_type, - null as commodity_name, - 'placeholder' as o_facility, - 'placeholder' as d_facility, - null as o_vertex_id, - null as d_vertex_id, - null as from_node_id, - null as to_node_id, - null as time_period, - null as commodity_id, - null as subcommodity_id, - null as source_facility_id, - null as source_facility_name, - null as units, - variable_name, - null as nx_edge_id, - null as mode, - null as mode_oid, - null as miles, - null as original_facility, - null as final_facility, - null as prior_edge - from optimal_solution - where variable_name like 'Build%'; - """ - db_con.execute("drop table if exists optimal_variables;") - db_con.execute(sql) - - - - # query the optimal_solution table in the DB for each variable we care about - #---------------------------------------------------------------------------- - sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_processors_count = data.fetchone()[0] - logger.info("number of optimal_processors: {}".format(optimal_processors_count)) - - sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_count = data.fetchone()[0] - logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) - sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmet_demand_sum = data.fetchone()[0] - logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) - logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) - logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format(optimal_unmet_demand_sum*the_scenario.unMetDemandPenalty)) - logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format(float(value(prob.objective)) - optimal_unmet_demand_sum*the_scenario.unMetDemandPenalty)) - logger.info("Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:," - ".0f}".format(float(value(prob.objective)))) - - sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" - data = db_con.execute(sql) - optimal_edges_count = data.fetchone()[0] - logger.info("number of optimal edges: {}".format(optimal_edges_count)) - - start_time = datetime.datetime.now() - logger.info("FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - -#------------------------------------------------------------------------------ - - -def parse_optimal_solution_db(the_scenario, logger): - logger.info("starting parse_optimal_solution") - - - optimal_processors = [] - optimal_processor_flows = [] - optimal_route_flows = {} - optimal_unmet_demand = {} - optimal_storage_flows = {} - optimal_excess_material = {} - vertex_id_to_facility_id_dict = {} - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # do the Storage Edges - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" - data = db_con.execute(sql) - optimal_storage_edges = data.fetchall() - for edge in optimal_storage_edges: - optimal_storage_flows[edge] = optimal_storage_edges[edge] - - # do the Route Edges - sql = """select - variable_name, variable_value, - cast(substr(variable_name, 6) as int) edge_id, - route_ID, start_day time_period, edges.commodity_id, - o_vertex_id, d_vertex_id, - v1.facility_id o_facility_id, - v2.facility_id d_facility_id - from optimal_solution - join edges on edges.edge_id = cast(substr(variable_name, 6) as int) - join vertices v1 on edges.o_vertex_id = v1.vertex_id - join vertices v2 on edges.d_vertex_id = v2.vertex_id - where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; - """ - data = db_con.execute(sql) - optimal_route_edges = data.fetchall() - for edge in optimal_route_edges: - - variable_name = edge[0] - - variable_value = edge[1] - - edge_id = edge[2] - - route_ID = edge[3] - - time_period = edge[4] - - commodity_flowed = edge[5] - - od_pair_name = "{}, {}".format(edge[8], edge[9]) - - - if route_ID not in optimal_route_flows: # first time route_id is used on a day or commodity - optimal_route_flows[route_ID] = [[od_pair_name, time_period, commodity_flowed, variable_value]] - - else: # subsequent times route is used on different day or for other commodity - optimal_route_flows[route_ID].append([od_pair_name, time_period, commodity_flowed, variable_value]) - - # do the processors - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" - data = db_con.execute(sql) - optimal_candidates_processors = data.fetchall() - for proc in optimal_candidates_processors: - optimal_processors.append(proc) - - - # do the processor vertex flows - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'ProcessorVertexFlow%';" - data = db_con.execute(sql) - optimal_processor_flows_sql = data.fetchall() - for proc in optimal_processor_flows_sql: - optimal_processor_flows.append(proc) - # optimal_biorefs.append(v.name[22:(v.name.find(",")-1)])# find the name from the - - # do the UnmetDemand - sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" - data = db_con.execute(sql) - optimal_unmetdemand = data.fetchall() - for ultimate_destination in optimal_unmetdemand: - v_name = ultimate_destination[0] - v_value = ultimate_destination[1] - - search = re.search('\(.*\)', v_name.replace("'", "")) - - if search: - parts = search.group(0).replace("(", "").replace(")", "").split(",_") - - dest_name = parts[0] - commodity_flowed = parts[2] - if not dest_name in optimal_unmet_demand: - optimal_unmet_demand[dest_name] = {} - - if not commodity_flowed in optimal_unmet_demand[dest_name]: - optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) - else: - optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) - - logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors - logger.info("length of optimal_processor_flows list: {}".format(len(optimal_processor_flows))) # a list of optimal processor flows - logger.info("length of optimal_route_flows dict: {}".format(len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values - logger.info("length of optimal_unmet_demand dict: {}".format(len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values - - return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material + +#--------------------------------------------------------------------------------------------------- +# Name: aftot_pulp_sasc +# +# Purpose: PulP optimization - source facility as subcommodity variant +# +#--------------------------------------------------------------------------------------------------- + +import os +import sys + +import ftot_supporting +import ftot_pulp +import sqlite3 +import re +import pdb + +import logging +import datetime + +from ftot import ureg, Q_ + +from collections import defaultdict + +from pulp import * +from ftot_supporting import get_total_runtime_string +#import ftot_supporting_gis +from six import iteritems + +#=================== constants============= +storage = 1 +primary = 0 +fixed_route_max_daily_capacity = 10000000000 +fixed_route_min_daily_capacity = 0 +fixed_schedule_id = 2 +fixed_route_duration = 0 +THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 + + +candidate_processing_facilities = [] + +storage_cost_1 = 0.01 +storage_cost_2 = 0.05 +facility_onsite_storage_max = 10000000000 +facility_onsite_storage_min = 0 +storage_cost_1 = .01 +storage_cost_2 = .05 +fixed_route_max_daily_capacity = 10000000000 +fixed_route_min_daily_capacity = 0 +default_max_capacity = 10000000000 +default_min_capacity = 0 + +#at this level, duplicate edges (i.e. same nx_edge_id, same subcommodity) are only created if they are lower cost. +duplicate_edge_cap = 1 + +#while loop cap - only used for dev to keep runtime reasonable in testing +while_loop_cap = 40 +#debug only +max_transport_distance = 50 + + +def o_sourcing(the_scenario, logger): + import ftot_pulp + pre_setup_pulp_from_optimal(logger, the_scenario) + prob = setup_pulp_problem(the_scenario, logger) + prob = solve_pulp_problem(prob, the_scenario, logger) + save_pulp_solution(the_scenario, prob, logger) + + from ftot_supporting import post_optimization + post_optimization(the_scenario, 'os', logger) + + +#=============================================================================== + + +def delete_all_global_dictionaries(): + global processing_facilities + global processor_storage_vertices + global supply_storage_vertices + global demand_storage_vertices + global processor_vertices + global supply_vertices + global demand_vertices + global storage_facility_vertices + global connector_edges + global storage_edges + global transport_edges + + + processing_facilities = [] + processor_storage_vertices = {} + supply_storage_vertices = {} + demand_storage_vertices = {} + # storage_facility_storage_vertices = {} + processor_vertices = {} + supply_vertices = {} + demand_vertices = {} + storage_facility_vertices = {} + connector_edges = {} + storage_edges = {} + transport_edges = {} + fuel_subtypes = {} + + +def save_existing_solution(logger, the_scenario): + logger.info("START: save_existing_solution") + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + logger.debug("save the current optimal_variables table as optimal_variables_prior") + main_db_con.executescript(""" + drop table if exists optimal_variables_prior; + + create table if not exists optimal_variables_prior as select * from optimal_variables; + + drop table if exists optimal_variables; + """) + + logger.debug("save the current vertices table as vertices_prior and create the new vertices table") + + main_db_con.executescript( + """ + drop table if exists vertices_prior; + + create table if not exists vertices_prior as select * from vertices; + """) + + +#=============================================================================== +def source_as_subcommodity_setup(the_scenario, logger): + logger.info("START: source_as_subcommodity_setup") + multi_commodity_name = "multicommodity" + with sqlite3.connect(the_scenario.main_db) as main_db_con: + main_db_con.executescript(""" + drop table if exists subcommodity + ; + + create table subcommodity(sub_id INTEGER PRIMARY KEY, + source_facility_id integer, + source_facility_name text, + commodity_id integer, + commodity_name text, + units text, + phase_of_matter text, + max_transport_distance text, + CONSTRAINT unique_source_and_name UNIQUE(commodity_name, source_facility_id)) + ; + + + insert into subcommodity ( + source_facility_id, + source_facility_name, + commodity_id, + commodity_name, + units, + phase_of_matter, + max_transport_distance) + + select + f.facility_id, + f.facility_name, + c.commodity_id, + c.commodity_name, + c.units, + c.phase_of_matter, + c.max_transport_distance + + from commodities c, facilities f, facility_type_id ft + where f.facility_type_id = ft.facility_type_id + and ft.facility_type = 'raw_material_producer' + ;""" + ) + main_db_con.execute("""insert or ignore into commodities(commodity_name) values ('{}');""".format(multi_commodity_name)) + main_db_con.execute("""insert or ignore into subcommodity(source_facility_id, source_facility_name, commodity_name, commodity_id) + select sc.source_facility_id, sc.source_facility_name, c.commodity_name, c.commodity_id + from subcommodity sc, commodities c + where c.commodity_name = '{}' + """.format(multi_commodity_name)) + return +#=============================================================================== + +def generate_all_vertices_from_optimal_for_sasc(the_scenario, schedule_dict, schedule_length, logger): + logger.info("START: generate_all_vertices_from_optimal_for_sasc") + #this should be run instead of generate_all_vertices_table, not in addition + + total_potential_production = {} + multi_commodity_name = "multicommodity" + #dictionary items should have the form (vertex, edge_names_in, edge_names_out) + + storage_availability = 1 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + main_db_con.executescript( + """ + drop table if exists vertices; + + create table if not exists vertices ( + vertex_id INTEGER PRIMARY KEY, location_id, + facility_id integer, facility_name text, facility_type_id integer, schedule_day integer, + commodity_id integer, activity_level numeric, storage_vertex binary, + udp numeric, supply numeric, demand numeric, + subcommodity_id integer, + source_facility_id integer, + CONSTRAINT unique_vertex UNIQUE(facility_id, schedule_day, subcommodity_id, storage_vertex)); + + insert or ignore into commodities(commodity_name) values ('{}');""".format(multi_commodity_name)) + + main_db_con.execute("""insert or ignore into subcommodity(source_facility_id, source_facility_name, commodity_name, commodity_id) + select sc.source_facility_id, sc.source_facility_name, c.commodity_name, c.commodity_id + from subcommodity sc, commodities c + where c.commodity_name = '{}' + """.format(multi_commodity_name)) + + #for all facilities that are used in the optimal solution, retrieve facility and facility_commodity information + #each row is a facility_commodity entry then, will get at least 1 vertex; more for days and storage, less for processors? + + #for raw_material_suppliers + #-------------------------------- + + + db_cur = main_db_con.cursor() + db_cur4 = main_db_con.cursor() + counter = 0 + for row in db_cur.execute("select count(distinct facility_id) from facilities;"): + total_facilities = row[0] + + #create vertices for each facility + # facility_type can be "raw_material_producer", "ultimate_destination","processor"; get id from original vertices table for sasc + # use UNION (? check for sqlite) to get a non-repeat list of facilities + + for row_a in db_cur.execute("""select f.facility_id, facility_type, facility_name, location_id, f.facility_type_id, schedule_id + from facilities f, facility_type_id ft, optimal_variables_prior ov + where ignore_facility = '{}' + and f.facility_type_id = ft.facility_type_id + and f.facility_name = ov.o_facility + union + select f.facility_id, facility_type, facility_name, location_id, f.facility_type_id + from facilities f, facility_type_id ft, optimal_variables_prior ov + where ignore_facility = '{}' + and f.facility_type_id = ft.facility_type_id + and f.facility_name = ov.d_facility + ;""".format('false', 'false')): + db_cur2 = main_db_con.cursor() + facility_id = row_a[0] + facility_type = row_a[1] + facility_name = row_a[2] + facility_location_id = row_a[3] + facility_type_id = row_a[4] + schedule_id = row_a[5] + if counter % 10000 == 0: + logger.info("vertices created for {} facilities of {}".format(counter, total_facilities)) + for row_d in db_cur4.execute("select count(distinct vertex_id) from vertices;"): + logger.info('{} vertices created'.format(row_d[0])) + counter = counter + 1 + + + if facility_type == "processor": + + #each processor facility should have 1 input commodity with 1 storage vertex, 1 or more output commodities each with 1 storage vertex, and 1 primary processing vertex + #explode by time, create primary vertex, explode by commodity to create storage vertices; can also create primary vertex for input commodity + #do this for each subcommodity now instead of each commodity + for row_b in db_cur2.execute("""select fc.commodity_id, + ifnull(fc.quantity, 0), + fc.units, + ifnull(c.supertype, c.commodity_name), + fc.io, + mc.commodity_id, + c.commodity_name, + s.sub_id, + s.source_facility_id + from facility_commodities fc, commodities c, commodities mc, subcommodity s + where fc.facility_id = {} + and fc.commodity_id = c.commodity_id + and mc.commodity_name = '{}' + and s.commodity_id = c.commodity_id;""".format(facility_id, multi_commodity_name)): + + commodity_id = row_b[0] + quantity = row_b[1] + units = row_b[2] + commodity_supertype = row_b[3] + io = row_b[4] + id_for_mult_commodities = row_b[5] + commodity_name = row_b[6] + subcommodity_id = row_b[7] + source_facility_id = row_b[8] + #vertices for generic demand type, or any subtype specified by the destination + for day_before, availability in enumerate(schedule_dict[schedule_id]): + if io == 'i': + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, + {},{} + );""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, id_for_mult_commodities, availability, primary, + subcommodity_id, source_facility_id)) + + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {}, {});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, + subcommodity_id, source_facility_id)) + + else: + if commodity_name != 'total_fuel': + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, {}, {});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, subcommodity_id, source_facility_id)) + + elif facility_type == "raw_material_producer": #raw material producer + + # handle as such, exploding by time and commodity + for row_b in db_cur2.execute("""select fc.commodity_id, fc.quantity, fc.units, s.sub_id, s.source_facility_id + from facility_commodities fc, subcommodity s + where fc.facility_id = {} + and s.source_facility_id = {} + and fc.commodity_id = s.commodity_id;""".format(facility_id, facility_id)): + commodity_id = row_b[0] + quantity = row_b[1] + units = row_b[2] + subcommodity_id = row_b[3] + source_facility_id = row_b[4] + + if commodity_id in total_potential_production: + total_potential_production[commodity_id] = total_potential_production[commodity_id] + quantity + else: + total_potential_production[commodity_id] = quantity + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, availability, primary, quantity, + subcommodity_id, source_facility_id)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, supply, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, + subcommodity_id, source_facility_id)) + + elif facility_type == "storage": #storage facility + # handle as such, exploding by time and commodity + for row_b in db_cur2.execute("""select commodity_id, quantity, units, s.sub_id, s.source_facility_id + from facility_commodities fc, subcommodity s + where fc.facility_id = {} + and fc.commodity_id = s.commodity_id;""".format(facility_id)): + commodity_id = row_b[0] + quantity = row_b[1] + units = row_b[2] + subcommodity_id = row_b[3] + source_facility_id = row_b[4] + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, storage_vertex, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage, + subcommodity_id, source_facility_id)) + + elif facility_type == "ultimate_destination": #ultimate_destination + # handle as such, exploding by time and commodity + for row_b in db_cur2.execute("""select fc.commodity_id, ifnull(fc.quantity, 0), fc.units, + fc.commodity_id, ifnull(c.supertype, c.commodity_name), s.sub_id, s.source_facility_id + from facility_commodities fc, commodities c, subcommodity s + where fc.facility_id = {} + and fc.commodity_id = c.commodity_id + and fc.commodity_id = s.commodity_id;""".format(facility_id)): + commodity_id = row_b[0] + quantity = row_b[1] + units = row_b[2] + commodity_id = row_b[3] + commodity_supertype = row_b[4] + subcommodity_id = row_b[5] + source_facility_id = row_b[6] + + #vertices for generic demand type, or any subtype specified by the destination + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, udp, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, + {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, + commodity_id, availability, primary, quantity, the_scenario.unMetDemandPenalty, + subcommodity_id, source_facility_id)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, commodity_id, storage_availability, storage, quantity, + subcommodity_id, source_facility_id)) + + #vertices for other fuel subtypes that match the destination's supertype + #if the subtype is in the commodity table, it is produced by some facility (not ignored) in the scenario + db_cur3 = main_db_con.cursor() + for row_c in db_cur3.execute("""select commodity_id, units from commodities + where supertype = '{}';""".format(commodity_supertype)): + new_commodity_id = row_c[0] + new_units= row_c[1] + for day_before, availability in enumerate(schedule_dict[schedule_id]): + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, udp, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, new_commodity_id, availability, primary, quantity, the_scenario.unMetDemandPenalty, + subcommodity_id, source_facility_id)) + main_db_con.execute("""insert or ignore into vertices ( + location_id, facility_id, facility_type_id, facility_name, schedule_day, commodity_id, activity_level, storage_vertex, demand, + subcommodity_id, source_facility_id) + values ({}, {}, {}, '{}', {}, {}, {}, {}, {}, + {},{});""".format(facility_location_id, facility_id, facility_type_id, facility_name, day_before+1, new_commodity_id, storage_availability, storage, quantity, + subcommodity_id, source_facility_id)) + + + else: + logger.warning("error, unexpected facility_type: {}, facility_type_id: {}".format(facility_type, facility_type_id)) + + # if it's an origin/supply facility, explode by commodity and time + # if it's a destination/demand facility, explode by commodity and time + # if it's an independent storage facility, explode by commodity and time + # if it's a processing/refining facility, only explode by time - all commodities on the product slate must + # enter and exit the vertex + + #add destination storage vertices for all demand subtypes after all facilities have been iterated, so that we know what fuel subtypes are in this scenario + + + logger.debug("total possible production in scenario: {}".format(total_potential_production)) + +#=============================================================================== + +def add_storage_routes(the_scenario, logger): + # these are loops to and from the same facility; when exploded to edges, they will connect primary to storage vertices, and storage vertices day to day + # is one enough? how many edges will be created per route here? + # will always create edge for this route from storage to storage vertex + # will always create edges for extra routes connecting here + # IF a primary vertex exists, will also create an edge connecting the storage vertex to the primary + + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + logger.debug("create the storage_routes table") + + main_db_con.execute("drop table if exists storage_routes;") + main_db_con.execute("""create table if not exists storage_routes as + select facility_name || '_storage' as route_name, + location_id, + facility_id, + facility_name as o_name, + facility_name as d_name, + {} as cost_1, + {} as cost_2, + 1 as travel_time, + {} as storage_max, + 0 as storage_min + from facilities + where ignore_facility = 'false' + ;""".format(storage_cost_1, storage_cost_2, facility_onsite_storage_max)) + main_db_con.execute("""create table if not exists route_reference( + route_id INTEGER PRIMARY KEY, route_type text, route_name text, scenario_rt_id integer, + CONSTRAINT unique_routes UNIQUE(route_type, route_name, scenario_rt_id));""") + #main_db_con.execute("insert or ignore into route_reference select scenario_rt_id, 'transport', 'see scenario_rt_id', scenario_rt_id from routes;") + main_db_con.execute("insert or ignore into route_reference select null,'storage', route_name, 0 from storage_routes;") + + + logger.debug("storage_routes table created") + + return + +#=============================================================================== +def generate_all_edges_from_optimal_for_sasc(the_scenario, schedule_length, logger): + logger.info("START: generate_all_edges_from_optimal_for_sasc") + #create edges table + #plan to generate start and end days based on nx edge time to traverse and schedule + #can still have route_id, but only for storage routes now; nullable + + multi_commodity_name = "multicommodity" + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + if ('pipeline_crude_trf_rts' in the_scenario.permittedModes) or ('pipeline_prod_trf_rts' in the_scenario.permittedModes) : + logger.info("create indices for the capacity_nodes and pipeline_mapping tables") + main_db_con.executescript( + """ + CREATE INDEX IF NOT EXISTS pm_index ON pipeline_mapping (id, id_field_name, mapping_id_field_name, mapping_id); + CREATE INDEX IF NOT EXISTS cn_index ON capacity_nodes (source, id_field_name, source_OID); + """) + main_db_con.executescript( + """ + drop table if exists edges_prior; + + create table edges_prior as select * from edges; + + drop table if exists edges; + + create table edges (edge_id INTEGER PRIMARY KEY, + route_id integer, + from_node_id integer, + to_node_id integer, + start_day integer, + end_day integer, + commodity_id integer, + o_vertex_id integer, + d_vertex_id integer, + max_edge_capacity numeric, + volume numeric, + capac_minus_volume_zero_floor numeric, + min_edge_capacity numeric, + capacity_units text, + units_conversion_multiplier numeric, + edge_flow_cost numeric, + edge_flow_cost2 numeric, + edge_type text, + nx_edge_id integer, + mode text, + mode_oid integer, + miles numeric, + simple_mode text, + tariff_id numeric, + phase_of_matter text, + subcommodity_id integer, + source_facility_id integer + );""") + + + + db_cur = main_db_con.cursor() + counter = 0 + for row in db_cur.execute("select commodity_id from commodities where commodity_name = '{}';""".format(multi_commodity_name)): + id_for_mult_commodities = row[0] + for row in db_cur.execute("select count(*) from networkx_edges;"): + total_transport_routes = row[0] + + + + #create transport edges, only between storage vertices and nodes, based on networkx graph and optimal variables + #never touch primary vertices; either or both vertices can be null (or node-type) if it's a mid-route link + #iterate through nx edges: if neither node has a location, create 1 edge per viable commodity + #should also be per day, subject to nx edge schedule + #before creating an edge, check: commodity allowed by nx and max transport distance if not null + #will need nodes per day and commodity? or can I just check that with constraints? + #select data for transport edges + for row_a in db_cur.execute("""select + ne.edge_id, + ifnull(fn.location_id, 'NULL'), + ifnull(tn.location_id, 'NULL'), + ne.mode_source, + ifnull(nec.phase_of_matter_id, 'NULL'), + nec.route_cost, + ne.from_node_id, + ne.to_node_id, + nec.dollar_cost, + ne.miles, + ne.capacity, + ne.artificial, + ne.mode_source_oid + from networkx_edges ne, networkx_nodes fn, networkx_nodes tn, networkx_edge_costs nec, optimal_variables_prior ov + + where ne.from_node_id = fn.node_id + and ne.to_node_id = tn.node_id + and ne.edge_id = nec.edge_id + and ifnull(ne.capacity, 1) > 0 + and ne.edge_id = ov.nx_edge_id + ;"""): + + nx_edge_id = row_a[0] + from_location = row_a[1] + to_location = row_a[2] + mode = row_a[3] + phase_of_matter = row_a[4] + route_cost = row_a[5] + from_node = row_a[6] + to_node = row_a[7] + dollar_cost = row_a[8] + miles = row_a[9] + max_daily_capacity = row_a[10] + mode_oid = row_a[12] + simple_mode = row_a[3].partition('_')[0] + + + db_cur3 = main_db_con.cursor() + + if counter % 10000 == 0: + logger.info("edges created for {} transport links of {}".format(counter, total_transport_routes)) + for row_d in db_cur3.execute("select count(distinct edge_id) from edges;"): + logger.info('{} edges created'.format(row_d[0])) + counter = counter +1 + + tariff_id = 0 + if simple_mode == 'pipeline': + + #find tariff_ids + + sql="select mapping_id from pipeline_mapping where id = {} and id_field_name = 'source_OID' and source = '{}' and mapping_id is not null;".format(mode_oid, mode) + for tariff_row in db_cur3.execute(sql): + tariff_id = tariff_row[0] + + if mode in the_scenario.permittedModes: + + # Edges are placeholders for flow variables + # 4-17: if both ends have no location, iterate through viable commodities and days, create edge + # for all days (restrict by link schedule if called for) + # for all allowed commodities, as currently defined by link phase of matter + + + for day in range(1, schedule_length+1): + if (day+fixed_route_duration <= schedule_length): #if link is traversable in the timeframe + if (simple_mode != 'pipeline' or tariff_id >= 0): + #for allowed commodities + for row_c in db_cur3.execute("select commodity_id, sub_id, source_facility_id from subcommodity where phase_of_matter = '{}' group by commodity_id, sub_id, source_facility_id".format(phase_of_matter)): + db_cur4 = main_db_con.cursor() + commodity_id = row_c[0] + subcommodity_id = row_c[1] + source_facility_id = row_c[2] + #if from_location != 'NULL': + # logger.info("non-null from location nxedge {}, {}, checking for commodity {}".format(from_location,nx_edge_id, commodity_id)) + if(from_location == 'NULL' and to_location == 'NULL'): + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{},{}); + """.format(from_node, to_node, + day, day+fixed_route_duration, commodity_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id,phase_of_matter, subcommodity_id, source_facility_id)) + + elif(from_location != 'NULL' and to_location == 'NULL'): + #for each day and commodity, get the corresponding origin vertex id to include with the edge info + #origin vertex must not be "ultimate_destination + #transport link outgoing from facility - checking fc.io is more thorough than checking if facility type is 'ultimate destination' + #new for bsc, only connect to vertices with matching subcommodity_id (only limited for RMP vertices) + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, subcommodity_id)): + from_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, + min_edge_capacity,edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id,phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{},{}); + """.format(from_node, to_node, + day, day+fixed_route_duration, commodity_id, + from_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) + elif(from_location == 'NULL' and to_location != 'NULL'): + #for each day and commodity, get the corresponding destination vertex id to include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, subcommodity_id)): + to_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, + {}, {}, {}, + '{}',{},'{}',{},{},'{}',{},'{}',{},{}); + """.format(from_node, to_node, + day, day+fixed_route_duration, commodity_id, + to_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) + elif(from_location != 'NULL' and to_location != 'NULL'): + #for each day and commodity, get the corresponding origin and destination vertex ids to include with the edge info + for row_d in db_cur4.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'o'""".format(from_location, day, commodity_id, subcommodity_id)): + from_vertex_id = row_d[0] + db_cur5 = main_db_con.cursor() + for row_e in db_cur5.execute("""select vertex_id + from vertices v, facility_commodities fc + where v.location_id = {} and v.schedule_day = {} and v.commodity_id = {} and v.subcommodity_id = {} + and v.storage_vertex = 1 + and v.facility_id = fc.facility_id + and v.commodity_id = fc.commodity_id + and fc.io = 'i'""".format(to_location, day, commodity_id, subcommodity_id)): + to_vertex_id = row_d[0] + main_db_con.execute("""insert or ignore into edges (from_node_id, to_node_id, + start_day, end_day, commodity_id, + o_vertex_id, d_vertex_id, + min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id) VALUES ({}, {}, + {}, {}, {}, + {}, {}, + {}, {}, {}, + '{}',{},'{}', {},{},'{}',{},'{}',{},{}); + """.format(from_node, to_node, + day, day+fixed_route_duration, commodity_id, + from_vertex_id, to_vertex_id, + default_min_capacity, route_cost, dollar_cost, + 'transport', nx_edge_id, mode, mode_oid, miles, simple_mode, tariff_id, phase_of_matter, subcommodity_id, source_facility_id)) + + + + + for row_d in db_cur.execute("select count(distinct edge_id) from edges;"): + logger.info('{} edges created'.format(row_d[0])) + + logger.debug("all transport edges created") + + #create storage & connector edges + #for each storage route, get origin storage vertices; if storage vertex exists on day of origin + duration, create a storage edge + for row_a in db_cur.execute("select sr.route_name, o_name, d_name, cost_1,cost_2, travel_time, storage_max, storage_min, rr.route_id, location_id, facility_id from storage_routes sr, route_reference rr where sr.route_name = rr.route_name;"): + route_name = row_a[0] + origin_facility_name = row_a[1] + dest_facility_name = row_a[2] + storage_route_cost_1 = row_a[3] + storage_route_cost_2 = row_a[4] + storage_route_travel_time = row_a[5] + max_daily_storage_capacity = row_a[6] + min_daily_storage_capacity = row_a[7] + route_id = row_a[8] + facility_location_id = row_a[9] + facility_id = row_a[10] + + + db_cur2 = main_db_con.cursor() + #storage vertices as origin for this route + #storage routes have no commodity phase restriction so we don't need to check for it here + for row_b in db_cur2.execute("""select v.vertex_id, v.schedule_day, + v.commodity_id, v.storage_vertex, v.subcommodity_id, v.source_facility_id + from vertices v + where v.facility_id = {} + and v.storage_vertex = 1;""".format(facility_id)): + o_vertex_id = row_b[0] + o_schedule_day = row_b[1] + o_commodity_id = row_b[2] + o_storage_vertex = row_b[3] + o_subcommodity_id = row_b[4] + o_source_facility_id = row_b[5] + + + #matching dest vertex for storage - same commodity and facility name, iterate day + db_cur3 = main_db_con.cursor() + for row_c in db_cur3.execute("""select v.vertex_id, v.schedule_day, v.storage_vertex + from vertices v + where v.facility_id = {} + and v.schedule_day = {} + and v.commodity_id = {} + and v.storage_vertex = 1 + and v.vertex_id != {} + and v.subcommodity_id = {};""".format(facility_id, o_schedule_day+storage_route_travel_time, o_commodity_id, o_vertex_id, o_subcommodity_id)): + d_vertex_id = row_c[0] + d_schedule_day = row_c[1] + o_storage_vertex = row_b[2] + main_db_con.execute("""insert or ignore into edges (route_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) + VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( + route_id, o_schedule_day, d_schedule_day, o_commodity_id, o_vertex_id, + d_vertex_id, max_daily_storage_capacity, min_daily_storage_capacity, storage_route_cost_1, 'storage', o_subcommodity_id, o_source_facility_id)) + + #connector edges from storage into primary vertex - processors and destinations + #check commodity direction = i instead of and v.facility_type_id != 'raw_material_producer' + #12-8 upates to account for multi-commodity processor vertices + #subcommodity has to match even if multi commodity - keep track of different processor vertex copies + db_cur3 = main_db_con.cursor() + for row_c in db_cur3.execute("""select d.vertex_id, d.schedule_day + from vertices d, facility_commodities fc + where d.facility_id = {} + and d.schedule_day = {} + and (d.commodity_id = {} or d.commodity_id = {}) + and d.storage_vertex = 0 + and d.facility_id = fc.facility_id + and fc.commodity_id = {} + and fc.io = 'i' + and d.source_facility_id = {} + ; + """.format(facility_id, o_schedule_day, o_commodity_id, id_for_mult_commodities, o_commodity_id,o_source_facility_id)): + d_vertex_id = row_c[0] + d_schedule_day = row_c[1] + connector_cost = 0 + main_db_con.execute("""insert or ignore into edges (route_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) + VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( + route_id, o_schedule_day, d_schedule_day, o_commodity_id, o_vertex_id, + d_vertex_id, default_max_capacity, default_min_capacity, connector_cost, 'connector', o_subcommodity_id, o_source_facility_id)) + + + #create remaining connector edges + #primary to storage vertex - supplier, destination, processor; not needed for storage facilities + #same day, same commodity (unless processor), no cost; purpose is to separate control of flows + #into and out of the system from flows within the system (transport & storage) + #and v.facility_type_id != 'ultimate_destination' + #create connectors ending at storage + + + d_vertex_id = row_b[0] + d_schedule_day = row_b[1] + d_commodity_id = row_b[2] + d_storage_vertex = row_b[3] + d_subcommodity_id = row_b[4] + d_source_facility_id = row_b[5] + + + db_cur3 = main_db_con.cursor() + for row_c in db_cur3.execute("""select vertex_id, schedule_day from vertices v, facility_commodities fc + where v.facility_id = {} + and v.schedule_day = {} + and (v.commodity_id = {} or v.commodity_id = {}) + and v.storage_vertex = 0 + and v.facility_id = fc.facility_id + and fc.commodity_id = {} + and fc.io = 'o' + and v.source_facility_id = {} + ; + """.format(facility_id, d_schedule_day, d_commodity_id, id_for_mult_commodities, d_commodity_id,d_source_facility_id)): + o_vertex_id = row_c[0] + o_schedule_day = row_c[1] + connector_cost = 0 + main_db_con.execute("""insert or ignore into edges (route_id, start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_type, subcommodity_id, source_facility_id) + VALUES ({}, {}, {}, {}, {}, {}, {}, {}, {}, '{}',{},{});""".format( + route_id, o_schedule_day, d_schedule_day, d_commodity_id, + o_vertex_id, d_vertex_id, default_max_capacity, default_min_capacity, + connector_cost, 'connector', d_subcommodity_id, d_source_facility_id)) + + logger.info("all edges created") + logger.info("create an index for the edges table by nodes") + + sql = ("""CREATE INDEX IF NOT EXISTS edge_index ON edges ( + edge_id, route_id, from_node_id, to_node_id, commodity_id, + start_day, end_day, commodity_id, o_vertex_id, d_vertex_id, + max_edge_capacity, min_edge_capacity, edge_flow_cost, edge_flow_cost2, + edge_type, nx_edge_id, mode, mode_oid, miles, subcommodity_id, source_facility_id);""") + db_cur.execute(sql) + + return + +#=============================================================================== +def set_edge_capacity_and_volume(the_scenario, logger): + + logger.info("START: set_edge_capacity_and_volume") + + multi_commodity_name = "multicommodity" + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + + logger.debug("starting to record volume and capacity for non-pipeline edges") + + main_db_con.execute("update edges set volume = (select ifnull(ne.volume,0) from networkx_edges ne where ne.edge_id = edges.nx_edge_id ) where simple_mode in ('rail','road','water');") + main_db_con.execute("update edges set max_edge_capacity = (select ne.capacity from networkx_edges ne where ne.edge_id = edges.nx_edge_id) where simple_mode in ('rail','road','water');") + logger.debug("volume and capacity recorded for non-pipeline edges") + + logger.debug("starting to record volume and capacity for pipeline edges") +## + main_db_con.executescript("""update edges set volume = + (select l.background_flow + from pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where edges.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(edges.mode, l.source)>0) + where simple_mode = 'pipeline' + ; + + update edges set max_edge_capacity = + (select l.capac + from pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where edges.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(edges.mode, l.source)>0) + where simple_mode = 'pipeline' + ;""") + + logger.debug("volume and capacity recorded for pipeline edges") + logger.debug("starting to record units and conversion multiplier") + logger.debug("update edges set capacity_units") + + main_db_con.execute("""update edges + set capacity_units = + (case when simple_mode = 'pipeline' then 'kbarrels' + when simple_mode = 'road' then 'truckload' + when simple_mode = 'rail' then 'railcar' + when simple_mode = 'water' then 'barge' + else 'unexpected mode' end) + ;""") + + logger.debug("update edges set units_conversion_multiplier;") + + main_db_con.execute("""update edges + set units_conversion_multiplier = + (case when simple_mode = 'pipeline' and phase_of_matter = 'liquid' then {} + when simple_mode = 'road' and phase_of_matter = 'liquid' then {} + when simple_mode = 'road' and phase_of_matter = 'solid' then {} + when simple_mode = 'rail' and phase_of_matter = 'liquid' then {} + when simple_mode = 'rail' and phase_of_matter = 'solid' then {} + when simple_mode = 'water' and phase_of_matter = 'liquid' then {} + when simple_mode = 'water' and phase_of_matter = 'solid' then {} + else 1 end) + ;""".format(THOUSAND_GALLONS_PER_THOUSAND_BARRELS, + the_scenario.truck_load_liquid.magnitude, + the_scenario.truck_load_solid.magnitude, + the_scenario.railcar_load_liquid.magnitude, + the_scenario.railcar_load_solid.magnitude, + the_scenario.barge_load_liquid.magnitude, + the_scenario.barge_load_solid.magnitude, + )) + + logger.debug("units and conversion multiplier recorded for all edges; starting capacity minus volume") + + main_db_con.execute("""update edges + set capac_minus_volume_zero_floor = + max((select (max_edge_capacity - ifnull(volume,0)) where max_edge_capacity is not null),0) + where max_edge_capacity is not null + ;""") + + logger.debug("capacity minus volume (minimum set to zero) recorded for all edges") + + return + + +#=============================================================================== +def pre_setup_pulp_from_optimal(logger, the_scenario): + + logger.info("START: pre_setup_pulp") + + source_as_subcommodity_setup(the_scenario, logger) + + schedule_dict, schedule_length = ftot_pulp.generate_schedules(the_scenario, logger) + + generate_all_vertices_from_optimal_for_sasc(the_scenario, schedule_dict, schedule_length, logger) + + add_storage_routes(the_scenario, logger) + generate_all_edges_from_optimal_for_sasc(the_scenario, schedule_length, logger) + set_edge_capacity_and_volume(the_scenario, logger) + + logger.info("Edges generated for modes: {}".format(the_scenario.permittedModes)) + + + return + +#=============================================================================== + +def create_flow_vars(the_scenario, logger): + # all_edges is a list of strings to be the keys for LPVariable.dict + + # we have a table called edges. + # call helper method to get list of unique IDs from the Edges table. + # use a the rowid as a simple unique integer index + edge_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + edge_list_cur = db_cur.execute("""select edge_id--, commodity_id, start_day, subcommodity_id + from edges;""") + edge_list_data = edge_list_cur.fetchall() + counter = 0 + for row in edge_list_data: + if counter % 500000 == 0: + logger.info("processed {:,.0f} records. size of edge_list {:,.0f}".format(counter, sys.getsizeof(edge_list))) + counter += 1 + #create an edge for each commodity allowed on this link - this construction may change as specific commodity restrictions are added + #TODO4-18 add days, but have no scheduel for links currently + #running just with nodes for now, will add proper facility info and storage back soon + edge_list.append((row[0])) + + logger.debug("MNP DEBUG: start assign flow_var with edge_list") + + flow_var = LpVariable.dicts("Edge",edge_list,0,None) + logger.debug("MNP DEBUG: Size of flow_var: {:,.0f}".format(sys.getsizeof(flow_var))) + #delete edge_list_data + edge_list_data = [] + return flow_var + # flow_var is the flow on each arc, being determined; this can be defined any time after all_arcs is defined + +#=============================================================================== +def create_unmet_demand_vars(the_scenario, logger): + + demand_var_list = [] + #may create vertices with zero demand, but only for commodities that the facility has demand for at some point + + + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) top_level_commodity_name, v.udp + from vertices v, commodities c, facility_type_id ft, facilities f + where v.commodity_id = c.commodity_id + and ft.facility_type = "ultimate_destination" + and v.storage_vertex = 0 + and v.facility_type_id = ft.facility_type_id + and v.facility_id = f.facility_id + and f.ignore_facility = 'false' + group by v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name) + ;""".format('')): + #facility_id, day, and simplified commodity name + demand_var_list.append((row[0], row[1], row[2], row[3])) + + unmet_demand_var = LpVariable.dicts("UnmetDemand", demand_var_list, 0, None) + + return unmet_demand_var + +#=============================================================================== + +def create_candidate_processor_build_vars(the_scenario, logger): + processors_build_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("select f.facility_id from facilities f, facility_type_id ft where f.facility_type_id = ft.facility_type_id and facility_type = 'processor' and candidate = 1 and ignore_facility = 'false' group by facility_id;"): + #grab all candidate processor facility IDs + processors_build_list.append(row[0]) + + processor_build_var = LpVariable.dicts("BuildProcessor", processors_build_list,0,None, 'Binary') + + + + return processor_build_var +#=============================================================================== +def create_binary_processor_vertex_flow_vars(the_scenario): + processors_flow_var_list = [] + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row in db_cur.execute("""select v.facility_id, v.schedule_day + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 0 + group by v.facility_id, v.schedule_day;"""): + #facility_id, day + processors_flow_var_list.append((row[0], row[1])) + + processor_flow_var = LpVariable.dicts("ProcessorDailyFlow", processors_flow_var_list, 0,None, 'Binary') + + + return processor_flow_var +#=============================================================================== +def create_processor_excess_output_vars(the_scenario): + excess_var_list = [] + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + xs_cur = db_cur.execute(""" + select vertex_id, commodity_id + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and facility_type = 'processor' + and storage_vertex = 1;""") + #facility_id, day, and simplified commodity name + xs_data = xs_cur.fetchall() + for row in xs_data: + excess_var_list.append(row[0]) + + + excess_var = LpVariable.dicts("XS", excess_var_list, 0,None) + + + return excess_var +#=============================================================================== + +def create_constraint_unmet_demand(logger, the_scenario, prob, flow_var, unmet_demand_var): + + logger.debug("START: create_constraint_unmet_demand") + + + + #apply activity_level to get corresponding actual demand for var + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + # var has form(facility_name, day, simple_fuel) + #unmet demand commodity should be simple_fuel = supertype + logger.debug("MNP: DEBUG: length of unmet_demand_vars: {}".format(len(unmet_demand_var))) + + demand_met_dict = defaultdict(list) + actual_demand_dict = {} + + demand_met = [] + # want to specify that all edges leading into this vertex + unmet demand = total demand + #demand primary (non-storage) vertices + + db_cur = main_db_con.cursor() + #each row_a is a primary vertex whose edges in contributes to the met demand of var + #will have one row for each fuel subtype in the scenario + unmet_data = db_cur.execute("""select v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.subcommodity_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + from vertices v, commodities c, subcommodity sc, facility_type_id ft, facilities f, edges e + where v.facility_id = f.facility_id + and ft.facility_type = 'ultimate_destination' + and f.facility_type_id = ft.facility_type_id + and f.ignore_facility = 'false' + and v.facility_type_id = ft.facility_type_id + and v.storage_vertex = 0 + and c.commodity_id = v.commodity_id + and sc.sub_id = v.subcommodity_id + and e.d_vertex_id = v.vertex_id + group by v.vertex_id, v.commodity_id, + v.demand, ifnull(c.proportion_of_supertype, 1), ifnull(v.activity_level, 1), v.subcommodity_id, + v.facility_id, v.schedule_day, ifnull(c.supertype, c.commodity_name), v.udp, e.edge_id + ;""") + + unmet_data = unmet_data.fetchall() + for row_a in unmet_data: + + primary_vertex_id = row_a[0] + commodity_id = row_a[1] + var_full_demand = row_a[2] + proportion_of_supertype = row_a[3] + var_activity_level = row_a[4] + subc_id = row_a[5] + facility_id = row_a[6] + day = row_a[7] + top_level_commodity = row_a[8] + udp = row_a[9] + edge_id = row_a[10] + var_actual_demand = var_full_demand*var_activity_level + + #next get inbound edges and apply appropriate modifier proportion to get how much of var's demand they satisfy + demand_met_dict[(facility_id, day, top_level_commodity, udp)].append(flow_var[edge_id]*proportion_of_supertype) + actual_demand_dict[(facility_id, day, top_level_commodity, udp)]=var_actual_demand + + for var in unmet_demand_var: + if var in demand_met_dict: + #then there are some edges in + prob += lpSum(demand_met_dict[var]) == actual_demand_dict[var] - unmet_demand_var[var], "constraint set unmet demand variable for facility {}, day {}, commodity {}".format(var[0], var[1], var[2]) + else: + #no edges in, so unmet demand equals full demand + prob += actual_demand_dict[var] == unmet_demand_var[var], "constraint set unmet demand variable for facility {}, day {}, commodity {} - no edges able to meet demand".format(var[0], var[1], var[2]) + + + + logger.debug("FINISHED: create_constraint_unmet_demand and return the prob ") + return prob + +#=============================================================================== +# create_constraint_max_flow_out_of_supply_vertex +# primary vertices only +# flow out of a vertex <= supply of the vertex, true for every day and commodity +def create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_max_flow_out_of_supply_vertex") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + + #for each primary (non-storage) supply vertex + #Assumption - each RMP produces a single commodity - + #Assumption - only one vertex exists per day per RMP (no multi commodity or subcommodity) + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + for row_a in db_cur.execute("""select vertex_id, activity_level, supply + from vertices v, facility_type_id ft + where v.facility_type_id = ft.facility_type_id + and ft.facility_type = 'raw_material_producer' + and storage_vertex = 0;"""): + supply_vertex_id = row_a[0] + activity_level = row_a[1] + max_daily_supply = row_a[2] + actual_vertex_supply = activity_level*max_daily_supply + + flow_out = [] + db_cur2 = main_db_con.cursor() + #select all edges leaving that vertex and sum their flows + #should be a single connector edge + for row_b in db_cur2.execute("select edge_id from edges where o_vertex_id = {};".format(supply_vertex_id)): + edge_id = row_b[0] + flow_out.append(flow_var[edge_id]) + + prob += lpSum(flow_out) <= actual_vertex_supply, "constraint max flow of {} out of origin vertex {}".format(actual_vertex_supply, supply_vertex_id) + #could easily add human-readable vertex info to this if desirable + + logger.debug("FINISHED: create_constraint_max_flow_out_of_supply_vertex") + return prob + #force flow out of origins to be <= supply + +#=============================================================================== + + +# for all of these vertices, flow in always == flow out +def create_processor_constraints(logger, the_scenario, prob, flow_var, processor_excess_vars, processor_build_vars, processor_daily_flow_vars): + logger.debug("STARTING: create_processor_constraints - capacity and conservation of flow") + node_counter = 0 + node_constraint_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + + logger.debug("conservation of flow, primary processor vertices:") + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + ifnull(f.candidate, 0) candidate_check, + e.subcommodity_id, + e.source_facility_id, + v.subcommodity_id, + v.source_facility_id + from vertices v, facility_commodities fc, facility_type_id ft, commodities c, facilities f + join edges e on (v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) + where ft.facility_type = 'processor' + and v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 0 + and v.facility_id = fc.facility_id + and fc.commodity_id = c.commodity_id + and fc.commodity_id = e.commodity_id + group by v.vertex_id, + in_or_out_edge, + constraint_day, + e.commodity_id, + e.mode, + e.edge_id, + nx_edge_id, fc.quantity, v.facility_id, c.commodity_name, + fc.io, + v.activity_level, + candidate_check, + e.subcommodity_id, + e.source_facility_id, + v.subcommodity_id, + v.source_facility_id + order by v.facility_id, e.source_facility_id, v.vertex_id, fc.io, e.edge_id + ;""" + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + sql_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info("execute for processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + sql_data = sql_data.fetchall() + logger.info("fetchall processor primary vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + + #keys are processor vertices + flow_in = {} + flow_out_lists = {} + + + for row_a in sql_data: + + vertex_id = row_a[0] + in_or_out_edge = row_a[1] + constraint_day = row_a[2] + commodity_id = row_a[3] + mode = row_a[4] + edge_id = row_a[5] + nx_edge_id = row_a[6] + quantity = float(row_a[7]) + facility_id = row_a[8] + commodity_name = row_a[9] + fc_io_commodity = row_a[10] + activity_level = row_a[11] + is_candidate = row_a[12] + edge_subcommodity_id = row_a[13] + edge_source_facility_id = row_a[14] + vertex_subcommodity_id = row_a[15] + vertex_source_facility_id = row_a[16] + + #edges should only connect to vertices with matching source facility + #if that's not true there could be issues here + + + if in_or_out_edge == 'in': + flow_in[vertex_id] = (flow_var[edge_id], quantity, commodity_name, activity_level, is_candidate, facility_id, vertex_source_facility_id) + elif in_or_out_edge == 'out': + flow_out_lists[(vertex_id, commodity_id)] = (flow_var[edge_id], quantity, commodity_name, facility_id, vertex_source_facility_id) + + + #require ratio to match for each output commodityalso do processor capacity constraint here + + for key, value in iteritems(flow_out_lists): + flow_out = value[0] + out_quantity = value[1] + vertex_id = key[0] + out_commodity = value[2] + flow_in_var = flow_in[vertex_id][0] + flow_in_quant = flow_in[vertex_id][1] + flow_in_commodity = flow_in[vertex_id][2] + required_flow_out = flow_out/out_quantity + required_flow_in = flow_in_var/flow_in_quant + + + # Sort edges by commodity and whether they're going in or out of the node + + prob += required_flow_out == required_flow_in, "conservation of flow, processor vertex {}, processor facility {}, commodity {} to {}, source facility {}".format(vertex_id, facility_id, flow_in_commodity, out_commodity, vertex_source_facility_id) + + logger.debug("FINISHED: processor conservation of flow and ratio constraint") + + + logger.debug("FINISHED: create_processor_constraints") + return prob + +def create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_var, processor_excess_vars): + logger.debug("STARTING: create_constraint_conservation_of_flow") + node_counter = 0 + node_constraint_counter = 0 + storage_vertex_constraint_counter = 0 + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + + logger.info("conservation of flow, storage vertices:") + #storage vertices, any facility type + #these have at most one direction of transport edges, so no need to track mode + sql = """select v.vertex_id, + (case when e.o_vertex_id = v.vertex_id then 'out' when e.d_vertex_id = v.vertex_id then 'in' else 'error' end) in_or_out_edge, + (case when e.o_vertex_id = v.vertex_id then start_day when e.d_vertex_id = v.vertex_id then end_day else 0 end) constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id, v.facility_id, c.commodity_name, + v.activity_level, + ft.facility_type + + from vertices v, facility_type_id ft, commodities c, facilities f + join edges e on ((v.vertex_id = e.o_vertex_id or v.vertex_id = e.d_vertex_id) and (e.o_vertex_id = v.vertex_id or e.d_vertex_id = v.vertex_id) and v.commodity_id = e.commodity_id) + + where v.facility_id = f.facility_id + and ft.facility_type_id = v.facility_type_id + and storage_vertex = 1 + and v.commodity_id = c.commodity_id + + group by v.vertex_id, + in_or_out_edge, + constraint_day, + v.commodity_id, + e.edge_id, + nx_edge_id,v.facility_id, c.commodity_name, + v.activity_level + + order by v.facility_id, v.vertex_id, e.edge_id + ;""" + + # get the data from sql and see how long it takes. + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + vertexid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info("execute for storage vertices, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + vertexid_data = vertexid_data.fetchall() + logger.info("fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + + flow_in_lists = {} + flow_out_lists = {} + for row_v in vertexid_data: + vertex_id = row_v[0] + in_or_out_edge = row_v[1] + constraint_day = row_v[2] + commodity_id = row_v[3] + edge_id = row_v[4] + nx_edge_id = row_v[5] + facility_id = row_v[6] + commodity_name = row_v[7] + activity_level = row_v[8] + facility_type = row_v[9] + + if in_or_out_edge == 'in': + flow_in_lists.setdefault((vertex_id, commodity_name, constraint_day, facility_type), []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((vertex_id, commodity_name, constraint_day, facility_type), []).append(flow_var[edge_id]) + + logger.info("adding processor excess variabless to conservation of flow") + for key, value in iteritems(flow_out_lists): + vertex_id = key[0] + commodity_name = key[1] + day = key[2] + facility_type = key[3] + if facility_type == 'processor': + flow_out_lists.setdefault(key, []).append(processor_excess_vars[vertex_id]) + + + for key, value in iteritems(flow_out_lists): + + if key in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[key]), "conservation of flow, vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + else: + prob += lpSum(flow_out_lists[key]) == lpSum(0), "conservation of flow (zero out), vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + for key, value in iteritems(flow_in_lists): + + if key not in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum(0), "conservation of flow (zero in), vertex {}, commodity {}, day {}".format(key[0], key[1], key[2]) + storage_vertex_constraint_counter = storage_vertex_constraint_counter + 1 + + logger.info("total conservation of flow constraints created on nodes: {}".format(storage_vertex_constraint_counter)) + + + + logger.info("conservation of flow, nx_nodes:") + #non-vertex nodes, no facility type, connect 2 transport edges over matching days and commodities + #for each day, get all edges in and out of the node. Sort them by commodity and whether they're going in or out of the node + sql = """select nn.node_id, + (case when e.from_node_id = nn.node_id then 'out' when e.to_node_id = nn.node_id then 'in' else 'error' end) in_or_out_edge, + (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 end) constraint_day, + e.commodity_id, + ifnull(mode, 'NULL'), + e.edge_id, nx_edge_id, + miles, + (case when ifnull(nn.source, 'N') == 'intermodal' then 'Y' else 'N' end) intermodal_flag, + e.subcommodity_id + from networkx_nodes nn + join edges e on (nn.node_id = e.from_node_id or nn.node_id = e.to_node_id) + where nn.location_id is null + order by node_id, commodity_id, + (case when e.from_node_id = nn.node_id then start_day when e.to_node_id = nn.node_id then end_day else 0 end), + in_or_out_edge, e.subcommodity_id + ;""" + + # get the data from sql and see how long it takes. + logger.info("Starting the long step:") + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + nodeid_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for :") + logger.info("execute for nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + nodeid_data = nodeid_data.fetchall() + logger.info("fetchall nodes with no location id, with their in and out edges - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + + flow_in_lists = {} + flow_out_lists = {} + + for row_a in nodeid_data: + node_id = row_a[0] + in_or_out_edge = row_a[1] + constraint_day = row_a[2] + commodity_id = row_a[3] + mode = row_a[4] + edge_id = row_a[5] + nx_edge_id = row_a[6] + miles = row_a[7] + intermodal = row_a[8] + subcommodity_id = row_a[9] + + #if node is not intermodal, conservation of flow holds per mode; + #if intermodal, then across modes + if intermodal == 'N': + if in_or_out_edge == 'in': + flow_in_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day, mode), []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day, mode), []).append(flow_var[edge_id]) + else: + if in_or_out_edge == 'in': + flow_in_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day), []).append(flow_var[edge_id]) + elif in_or_out_edge == 'out': + flow_out_lists.setdefault((node_id, intermodal, subcommodity_id, constraint_day), []).append(flow_var[edge_id]) + + + for key, value in iteritems(flow_out_lists): + node_id = key[0] + intermodal_flag = key[1] + subcommodity_id = key[2] + day = key[3] + if len(key) == 5: + node_mode = key[4] + else: node_mode = 'intermodal' + if key in flow_in_lists: + prob += lpSum(flow_out_lists[key]) == lpSum(flow_in_lists[key]), "conservation of flow, nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + else: + prob += lpSum(flow_out_lists[key]) == lpSum(0), "conservation of flow (zero out), nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + for key, value in iteritems(flow_in_lists): + node_id = key[0] + intermodal_flag = key[1] + subcommodity_id = key[2] + day = key[3] + if len(key) == 5: + node_mode = key[4] + else: node_mode = 'intermodal' + + if key not in flow_out_lists: + prob += lpSum(flow_in_lists[key]) == lpSum(0), "conservation of flow (zero in), nx node {}, subcommodity {}, day {}, mode {}".format(node_id, subcommodity_id, day, node_mode) + node_constraint_counter = node_constraint_counter + 1 + + logger.info("total conservation of flow constraints created on nodes: {}".format(node_constraint_counter)) + + #Note: no consesrvation of flow for primary vertices for supply & demand - they have unique constraints + + logger.debug("FINISHED: create_constraint_conservation_of_flow") + + return prob + +# set route capacity by phase of matter - may connect multiple facilities, definitely multiple edges +#do storage capacity separately, even though it is also routes +def create_constraint_max_route_capacity(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_max_route_capacity") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) + #min_capacity_level must be a number from 0 to 1, inclusive + #min_capacity_level is only relevant when background flows are turned on + #it sets a floor to how much capacity can be reduced by volume. + #min_capacity_level = .25 means route capacity will never be less than 25% of full capacity, even if "volume" would otherwise restrict it further + #min_capacity_level = 0 allows a route to be made unavailable for FTOT flow if base volume is too high + #this currently applies to all modes; could be made mode specific + logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + #capacity for storage routes + sql = """select + rr.route_id, sr.storage_max, sr.route_name, e.edge_id, e.start_day + from route_reference rr + join storage_routes sr on sr.route_name = rr.route_name + join edges e on rr.route_id = e.route_id + ;""" + # get the data from sql and see how long it takes. + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + storage_edge_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for storage edges:") + logger.info("execute for edges for storage - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + storage_edge_data = storage_edge_data.fetchall() + logger.info("fetchall edges for storage - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in storage_edge_data: + route_id = row_a[0] + aggregate_storage_capac = row_a[1] + storage_route_name = row_a[2] + edge_id = row_a[3] + start_day = row_a[4] + + flow_lists.setdefault((route_id, aggregate_storage_capac, storage_route_name, start_day), []).append(flow_var[edge_id]) + + + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on storage route {} named {} for day {}".format(key[0], key[2], key[3]) + + logger.debug("route_capacity constraints created for all storage routes") + + + + #capacity for transport routes + #Assumption - all flowing material is in kgal, all flow is summed on a single non-pipeline nx edge + sql = """select e.edge_id, e.nx_edge_id, e.max_edge_capacity, e.start_day, e.simple_mode, e.phase_of_matter, e.capac_minus_volume_zero_floor + from edges e + where e.max_edge_capacity is not null + and e.simple_mode != 'pipeline' + ;""" + # get the data from sql and see how long it takes. + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + route_capac_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for transport edges:") + logger.info("execute for non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + route_capac_data = route_capac_data.fetchall() + logger.info("fetchall non-pipeline edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in route_capac_data: + edge_id = row_a[0] + nx_edge_id = row_a[1] + nx_edge_capacity = row_a[2] + start_day = row_a[3] + simple_mode = row_a[4] + phase_of_matter = row_a[5] + capac_minus_background_flow = max(row_a[6], 0) + min_restricted_capacity = max(capac_minus_background_flow, nx_edge_capacity*the_scenario.minCapacityLevel) + + if simple_mode in the_scenario.backgroundFlowModes: + use_capacity = min_restricted_capacity + else: use_capacity = nx_edge_capacity + + #flow is in thousand gallons (kgal), for liquid, or metric tons, for solid + #capacity is in truckload, rail car, barge, or pipeline movement per day + # if mode is road and phase is liquid, capacity is in truckloads per day, we want it in kgal + # ftot_supporting_gis tells us that there are 8 kgal per truckload, so capacity * 8 gives us correct units or kgal per day + # use capacity * ftot_supporting_gis multiplier to get capacity in correct flow units + + multiplier = 1 #unless otherwise specified + if simple_mode == 'road': + if phase_of_matter == 'liquid': + multiplier = the_scenario.truck_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.truck_load_solid.magnitude + elif simple_mode == 'water': + if phase_of_matter == 'liquid': + multiplier = the_scenario.barge_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.barge_load_solid.magnitude + elif simple_mode == 'rail': + if phase_of_matter == 'liquid': + multiplier = the_scenario.railcar_load_liquid.magnitude + elif phase_of_matter == 'solid': + multiplier = the_scenario.railcar_load_solid.magnitude + + converted_capacity = use_capacity*multiplier + + flow_lists.setdefault((nx_edge_id, converted_capacity, start_day), []).append(flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on nx edge {} for day {}".format(key[0], key[2]) + + logger.debug("route_capacity constraints created for all non-pipeline transport routes") + + + logger.debug("FINISHED: create_constraint_max_route_capacity") + return prob + +def create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_var): + logger.debug("STARTING: create_constraint_pipeline_capacity") + logger.debug("Length of flow_var: {}".format(len(list(flow_var.items())))) + logger.info("modes with background flow turned on: {}".format(the_scenario.backgroundFlowModes)) + logger.info("minimum available capacity floor set at: {}".format(the_scenario.minCapacityLevel)) + + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + + #capacity for pipeline tariff routes + #with sasc, may have multiple flows per segment, slightly diff commodities + sql = """select e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, l.capac-l.background_flow allowed_flow, l.source, e.mode, instr(e.mode, l.source) + from edges e, pipeline_mapping pm, + (select id_field_name, cn.source_OID as link_id, min(cn.capacity) capac, + max(cn.volume) background_flow, source + from capacity_nodes cn + where cn.id_field_name = 'MASTER_OID' + and ifnull(cn.capacity,0)>0 + group by link_id) l + + where e.tariff_id = pm.id + and pm.id_field_name = 'tariff_ID' + and pm.mapping_id_field_name = 'MASTER_OID' + and l.id_field_name = 'MASTER_OID' + and pm.mapping_id = l.link_id + and instr(e.mode, l.source)>0 + group by e.edge_id, e.tariff_id, l.link_id, l.capac, e.start_day, allowed_flow, l.source + ;""" + # capacity needs to be shared over link_id for any edge_id associated with that link + + logger.info("Starting the execute") + execute_start_time = datetime.datetime.now() + pipeline_capac_data = db_cur.execute(sql) + logger.info("Done with the execute fetch all for transport edges:") + logger.info("execute for edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(execute_start_time))) + + logger.info("Starting the fetchall") + fetchall_start_time = datetime.datetime.now() + pipeline_capac_data = pipeline_capac_data.fetchall() + logger.info("fetchall edges for transport edge capacity - Total Runtime (HMS): \t{} \t ".format(get_total_runtime_string(fetchall_start_time))) + + flow_lists = {} + + for row_a in pipeline_capac_data: + edge_id = row_a[0] + tariff_id = row_a[1] + link_id = row_a[2] + # Link capacity is recorded in "thousand barrels per day"; 1 barrel = 42 gall + # Link capacity * 42 is now in kgal per day, to match flow in kgal + link_capacity_kgal_per_day = THOUSAND_GALLONS_PER_THOUSAND_BARRELS*row_a[3] + start_day = row_a[4] + capac_minus_background_flow_kgal = max(THOUSAND_GALLONS_PER_THOUSAND_BARRELS*row_a[5],0) + min_restricted_capacity = max(capac_minus_background_flow_kgal, link_capacity_kgal_per_day*the_scenario.minCapacityLevel) + + capacity_nodes_mode_source = row_a[6] + edge_mode = row_a[7] + mode_match_check = row_a[8] + if 'pipeline' in the_scenario.backgroundFlowModes: + link_use_capacity = min_restricted_capacity + else: link_use_capacity = link_capacity_kgal_per_day + + #add flow from all relevant edges, for one start; may be multiple tariffs + flow_lists.setdefault((link_id, link_use_capacity, start_day, edge_mode), []).append(flow_var[edge_id]) + + for key, flow in iteritems(flow_lists): + prob += lpSum(flow) <= key[1], "constraint max flow on pipeline link {} for mode {} for day {}".format(key[0], key[3], key[2]) + + logger.debug("pipeline capacity constraints created for all transport routes") + + + logger.debug("FINISHED: create_constraint_pipeline_capacity") + return prob + +#=============================================================================== +## + +def setup_pulp_problem(the_scenario, logger): + + + logger.info("START: setup PuLP problem") + + # flow_var is the flow on each edge by commodity and day. + # the optimal value of flow_var will be solved by PuLP + logger.info("calling create_flow_vars") + flow_vars = create_flow_vars(the_scenario, logger) + + logger.info("calling create_unmet_demand_vars") + #unmet_demand_var is the unmet demand at each destination, being determined + unmet_demand_vars = create_unmet_demand_vars(the_scenario, logger) + + logger.info("calling create_candidate_processor_build_vars") + #processor_build_vars is the binary variable indicating whether a candidate processor is used and thus its build cost charged + processor_build_vars = create_candidate_processor_build_vars(the_scenario, logger) + + logger.info("calling create_binary_processor_vertex_flow_vars") + #binary tracker variables + processor_vertex_flow_vars = create_binary_processor_vertex_flow_vars(the_scenario) + + logger.info("calling create_processor_excess_output_vars") + #tracking unused production + processor_excess_vars = create_processor_excess_output_vars(the_scenario) + + + + # THIS IS THE OBJECTIVE FUCTION FOR THE OPTIMIZATION + # ================================================== + logger.info("calling create_opt_problem") + + prob = ftot_pulp.create_opt_problem(logger,the_scenario, unmet_demand_vars, flow_vars, processor_build_vars) + logger.info("MNP DEBUG: size of prob: {}".format(sys.getsizeof(prob))) + + logger.info("calling create_constraint_unmet_demand") + prob = create_constraint_unmet_demand(logger,the_scenario, prob, flow_vars, unmet_demand_vars) + logger.debug("MNP DEBUG: size of prob: {}".format(sys.getsizeof(prob))) + + logger.info("calling create_constraint_max_flow_out_of_supply_vertex") + prob = create_constraint_max_flow_out_of_supply_vertex(logger, the_scenario, prob, flow_vars) + + + logger.info("calling create_constraint_daily_processor_capacity from ftot_pulp") + from ftot_pulp import create_constraint_daily_processor_capacity + prob = create_constraint_daily_processor_capacity(logger, the_scenario, prob, flow_vars, processor_build_vars, processor_vertex_flow_vars) + + logger.info("calling create_processor_constraints") + prob = create_processor_constraints(logger, the_scenario, prob, flow_vars, processor_excess_vars, processor_build_vars, processor_vertex_flow_vars) + + logger.info("calling create_constraint_conservation_of_flow") + prob = create_constraint_conservation_of_flow(logger, the_scenario, prob, flow_vars, processor_excess_vars) + + if the_scenario.capacityOn: + + logger.info("calling create_constraint_max_route_capacity") + prob = create_constraint_max_route_capacity(logger, the_scenario, prob, flow_vars) + + logger.info("calling create_constraint_pipeline_capacity") + prob = create_constraint_pipeline_capacity(logger, the_scenario, prob, flow_vars) + + del(unmet_demand_vars) + + del(flow_vars) + + # SCENARIO SPECIFIC CONSTRAINTS + + logger.info("calling write LP file") + prob.writeLP(os.path.join(the_scenario.scenario_run_directory, "debug", "LP_output_sasc.lp")) + logger.info("FINISHED: setup PuLP problem") + + return prob + +###=============================================================================== +def solve_pulp_problem(prob_final, the_scenario, logger): + + import datetime + + logger.info("START: prob.solve()") + #begin new prob.solve logging, 9-04-2018 + start_time = datetime.datetime.now() + from os import dup, dup2, close + f = open(os.path.join(the_scenario.scenario_run_directory, "debug", 'probsolve_capture.txt'), 'w') + orig_std_out = dup(1) + dup2(f.fileno(), 1) + + #status = prob_final.solve (PULP_CBC_CMD(maxSeconds = i_max_sec, fracGap = d_opt_gap, msg=1)) # CBC time limit and relative optimality gap tolerance + status = prob_final.solve (PULP_CBC_CMD(msg=1)) # CBC time limit and relative optimality gap tolerance + print('Completion code: %d; Solution status: %s; Best obj value found: %s' % (status, LpStatus[prob_final.status], value(prob_final.objective))) + + dup2(orig_std_out, 1) + close(orig_std_out) + f.close() + #end new prob.solve logging, 9-04-2018 + + logger.info("completed calling prob.solve()") + logger.info("FINISH: prob.solve(): Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + # THIS IS THE SOLUTION + # The status of the solution is printed to the screen + ##LpStatus key string value numerical value + ##LpStatusOptimal ?Optimal? 1 + ##LpStatusNotSolved ?Not Solved? 0 + ##LpStatusInfeasible ?Infeasible? -1 + ##LpStatusUnbounded ?Unbounded? -2 + ##LpStatusUndefined ?Undefined? -3 + logger.result("prob.Status: \t {}".format(LpStatus[prob_final.status])) + + logger.result("Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:,.0f}".format(float(value(prob_final.objective)))) + + return prob_final +#------------------------------------------------------------------------------ +def save_pulp_solution(the_scenario, prob, logger): + import datetime + + logger.info("START: save_pulp_solution") + non_zero_variable_count = 0 + + + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_cur = db_con.cursor() + # drop the optimal_solution table + #----------------------------- + db_cur.executescript("drop table if exists optimal_solution;") + + # create the optimal_solution table + #----------------------------- + db_cur.executescript(""" + create table optimal_solution + ( + variable_name string, + variable_value real + ); + """) + + + # insert the optimal data into the DB + #------------------------------------- + for v in prob.variables(): + if v.varValue > 0.0: + + sql = """insert into optimal_solution (variable_name, variable_value) values ("{}", {});""".format(v.name, float(v.varValue)) + db_con.execute(sql) + non_zero_variable_count = non_zero_variable_count + 1 + + logger.info("number of solution variables greater than zero: {}".format(non_zero_variable_count)) + + sql = """ + create table optimal_variables as + select + 'UnmetDemand' as variable_type, + cast(substr(variable_name, 13) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + null as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as subcommodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as original_facility, + null as final_facility, + null as prior_edge + from optimal_solution + where variable_name like 'UnmetDemand%' + union + select + 'Edge' as variable_type, + cast(substr(variable_name, 6) as int) var_id, + variable_value, + edges.max_edge_capacity*edges.units_conversion_multiplier as converted_capacity, + edges.volume*edges.units_conversion_multiplier as converted_volume, + edges.capac_minus_volume_zero_floor*edges.units_conversion_multiplier as converted_capac_minus_volume, + edges.edge_type, + commodities.commodity_name, + ov.facility_name as o_facility, + dv.facility_name as d_facility, + o_vertex_id, + d_vertex_id, + from_node_id, + to_node_id, + start_day time_period, + edges.commodity_id, + edges.subcommodity_id, + edges.source_facility_id, + s.source_facility_name, + commodities.units, + variable_name, + edges.nx_edge_id, + edges.mode, + edges.mode_oid, + edges.miles, + null as original_facility, + null as final_facility, + null as prior_edge + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join commodities on edges.commodity_id = commodities.commodity_ID + left outer join vertices as ov on edges.o_vertex_id = ov.vertex_id + left outer join vertices as dv on edges.d_vertex_id = dv.vertex_id + left outer join subcommodity as s on edges.subcommodity_id = s.sub_id + where variable_name like 'Edge%' + union + select + 'BuildProcessor' as variable_type, + cast(substr(variable_name, 16) as int) var_id, + variable_value, + null as converted_capacity, + null as converted_volume, + null as converted_capac_minus_volume, + null as edge_type, + null as commodity_name, + 'placeholder' as o_facility, + 'placeholder' as d_facility, + null as o_vertex_id, + null as d_vertex_id, + null as from_node_id, + null as to_node_id, + null as time_period, + null as commodity_id, + null as subcommodity_id, + null as source_facility_id, + null as source_facility_name, + null as units, + variable_name, + null as nx_edge_id, + null as mode, + null as mode_oid, + null as miles, + null as original_facility, + null as final_facility, + null as prior_edge + from optimal_solution + where variable_name like 'Build%'; + """ + db_con.execute("drop table if exists optimal_variables;") + db_con.execute(sql) + + + + # query the optimal_solution table in the DB for each variable we care about + #---------------------------------------------------------------------------- + sql = "select count(variable_name) from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_processors_count = data.fetchone()[0] + logger.info("number of optimal_processors: {}".format(optimal_processors_count)) + + sql = "select count(variable_name) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_count = data.fetchone()[0] + logger.info("number facilities with optimal_unmet_demand : {}".format(optimal_unmet_demand_count)) + sql = "select ifnull(sum(variable_value),0) from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmet_demand_sum = data.fetchone()[0] + logger.info("Total Unmet Demand : {}".format(optimal_unmet_demand_sum)) + logger.info("Penalty per unit of Unmet Demand : ${0:,.0f}".format(the_scenario.unMetDemandPenalty)) + logger.info("Total Cost of Unmet Demand : \t ${0:,.0f}".format(optimal_unmet_demand_sum*the_scenario.unMetDemandPenalty)) + logger.info("Total Cost of building and transporting : \t ${0:,.0f}".format(float(value(prob.objective)) - optimal_unmet_demand_sum*the_scenario.unMetDemandPenalty)) + logger.info("Total Scenario Cost = (transportation + unmet demand penalty + processor construction): \t ${0:," + ".0f}".format(float(value(prob.objective)))) + + sql = "select count(variable_name) from optimal_solution where variable_name like 'Edge%';" + data = db_con.execute(sql) + optimal_edges_count = data.fetchone()[0] + logger.info("number of optimal edges: {}".format(optimal_edges_count)) + + start_time = datetime.datetime.now() + logger.info("FINISH: save_pulp_solution: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + +#------------------------------------------------------------------------------ + + +def parse_optimal_solution_db(the_scenario, logger): + logger.info("starting parse_optimal_solution") + + + optimal_processors = [] + optimal_processor_flows = [] + optimal_route_flows = {} + optimal_unmet_demand = {} + optimal_storage_flows = {} + optimal_excess_material = {} + vertex_id_to_facility_id_dict = {} + + with sqlite3.connect(the_scenario.main_db) as db_con: + + # do the Storage Edges + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'Edge%_storage';" + data = db_con.execute(sql) + optimal_storage_edges = data.fetchall() + for edge in optimal_storage_edges: + optimal_storage_flows[edge] = optimal_storage_edges[edge] + + # do the Route Edges + sql = """select + variable_name, variable_value, + cast(substr(variable_name, 6) as int) edge_id, + route_ID, start_day time_period, edges.commodity_id, + o_vertex_id, d_vertex_id, + v1.facility_id o_facility_id, + v2.facility_id d_facility_id + from optimal_solution + join edges on edges.edge_id = cast(substr(variable_name, 6) as int) + join vertices v1 on edges.o_vertex_id = v1.vertex_id + join vertices v2 on edges.d_vertex_id = v2.vertex_id + where variable_name like 'Edge%_' and variable_name not like 'Edge%_storage'; + """ + data = db_con.execute(sql) + optimal_route_edges = data.fetchall() + for edge in optimal_route_edges: + + variable_name = edge[0] + + variable_value = edge[1] + + edge_id = edge[2] + + route_ID = edge[3] + + time_period = edge[4] + + commodity_flowed = edge[5] + + od_pair_name = "{}, {}".format(edge[8], edge[9]) + + + if route_ID not in optimal_route_flows: # first time route_id is used on a day or commodity + optimal_route_flows[route_ID] = [[od_pair_name, time_period, commodity_flowed, variable_value]] + + else: # subsequent times route is used on different day or for other commodity + optimal_route_flows[route_ID].append([od_pair_name, time_period, commodity_flowed, variable_value]) + + # do the processors + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'BuildProcessor%';" + data = db_con.execute(sql) + optimal_candidates_processors = data.fetchall() + for proc in optimal_candidates_processors: + optimal_processors.append(proc) + + + # do the processor vertex flows + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'ProcessorVertexFlow%';" + data = db_con.execute(sql) + optimal_processor_flows_sql = data.fetchall() + for proc in optimal_processor_flows_sql: + optimal_processor_flows.append(proc) + # optimal_biorefs.append(v.name[22:(v.name.find(",")-1)])# find the name from the + + # do the UnmetDemand + sql = "select variable_name, variable_value from optimal_solution where variable_name like 'UnmetDemand%';" + data = db_con.execute(sql) + optimal_unmetdemand = data.fetchall() + for ultimate_destination in optimal_unmetdemand: + v_name = ultimate_destination[0] + v_value = ultimate_destination[1] + + search = re.search('\(.*\)', v_name.replace("'", "")) + + if search: + parts = search.group(0).replace("(", "").replace(")", "").split(",_") + + dest_name = parts[0] + commodity_flowed = parts[2] + if not dest_name in optimal_unmet_demand: + optimal_unmet_demand[dest_name] = {} + + if not commodity_flowed in optimal_unmet_demand[dest_name]: + optimal_unmet_demand[dest_name][commodity_flowed] = int(v_value) + else: + optimal_unmet_demand[dest_name][commodity_flowed] += int(v_value) + + logger.info("length of optimal_processors list: {}".format(len(optimal_processors))) # a list of optimal processors + logger.info("length of optimal_processor_flows list: {}".format(len(optimal_processor_flows))) # a list of optimal processor flows + logger.info("length of optimal_route_flows dict: {}".format(len(optimal_route_flows))) # a dictionary of routes keys and commodity flow values + logger.info("length of optimal_unmet_demand dict: {}".format(len(optimal_unmet_demand))) # a dictionary of route keys and unmet demand values + + return optimal_processors, optimal_route_flows, optimal_unmet_demand, optimal_storage_flows, optimal_excess_material diff --git a/program/ftot_report.py b/program/ftot_report.py index 0e6d6cc..9dec6cf 100644 --- a/program/ftot_report.py +++ b/program/ftot_report.py @@ -1,509 +1,609 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot_report -# -# Purpose: parses the most current log file for each step and assembles two reports. -# the first is a human readable report that groups log messages by runtime, result, config, and warnings. -# The second report is formatted for FTOT tableau dashboard as a simple CSV and comes from a few tables in the db. -# -# --------------------------------------------------------------------------------------------------- - -import os -import datetime -import glob -import ntpath -import zipfile -from ftot_supporting_gis import zipgdb -from ftot_supporting import clean_file_name -from shutil import copy - -from ftot import FTOT_VERSION - -# ================================================================== - - -def prepare_tableau_assets(report_file, the_scenario, logger): - logger.info("start: prepare_tableau_assets") - timestamp_folder_name = 'tableau_report_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") - report_directory = os.path.join(the_scenario.scenario_run_directory, "Reports", timestamp_folder_name) - if not os.path.exists(report_directory): - os.makedirs(report_directory) - - # make tableau_output.gdb file - # ----------------------------------- - logger.debug("start: create tableau gdb") - import arcpy - arcpy.CreateFileGDB_management(out_folder_path=report_directory, - out_name="tableau_output", out_version="CURRENT") - - # add Dataset field (raw_material_producers, processors, ultimate_destinations) - arcpy.AddField_management(the_scenario.rmp_fc, "Dataset", "TEXT") - arcpy.CalculateField_management(in_table=the_scenario.rmp_fc, field="Dataset", - expression='"raw_material_producers"', - expression_type="PYTHON_9.3", code_block="") - arcpy.AddField_management(the_scenario.processors_fc, "Dataset", "TEXT") - arcpy.CalculateField_management(in_table=the_scenario.processors_fc, field="Dataset", - expression='"processors"', - expression_type="PYTHON_9.3", code_block="") - arcpy.AddField_management(the_scenario.destinations_fc, "Dataset", "TEXT") - arcpy.CalculateField_management(in_table=the_scenario.destinations_fc, field="Dataset", - expression='"ultimate_destinations"', - expression_type="PYTHON_9.3", code_block="") - - # merge facility type FCs - facilities_merge_fc = os.path.join(report_directory, "tableau_output.gdb", "facilities_merge") - arcpy.Merge_management([the_scenario.destinations_fc, the_scenario.rmp_fc, the_scenario.processors_fc], - facilities_merge_fc) - - # add the scenario_name in the facilities_merge - arcpy.AddField_management(facilities_merge_fc, - "Scenario_Name", - "TEXT") - arcpy.CalculateField_management(in_table=facilities_merge_fc, field="Scenario_Name", - expression='"{}".format(the_scenario.scenario_name)', - expression_type="PYTHON_9.3", code_block="") - - - - # copy optimized_route_segments_disolved (aka: ORSD) - output_ORSD = os.path.join(report_directory, "tableau_output.gdb", "optimized_route_segments_dissolved") - arcpy.Copy_management( - in_data=os.path.join(the_scenario.main_gdb, "optimized_route_segments_dissolved"), - out_data=output_ORSD, - data_type="FeatureClass") - # add field "record_id" - arcpy.AddField_management(output_ORSD, "record_id", "TEXT") - - # field calculator; netsource + netsource_oid for unique field - arcpy.CalculateField_management(in_table=output_ORSD, field="record_id", - expression='"{}_{} ".format(!OBJECTID!, !NET_SOURCE_NAME!)', - expression_type="PYTHON_9.3", code_block="") - - # add the scenario_name in the optimized_route_segments_dissolved_fc (output_ORSD) - arcpy.AddField_management(output_ORSD, - "Scenario_Name", - "TEXT") - arcpy.CalculateField_management(in_table=output_ORSD, field="Scenario_Name", - expression='"{}".format(the_scenario.scenario_name)', - expression_type="PYTHON_9.3", code_block="") - - # copy optimized_route_segments (ORS) - # this contains commodity info at the link level - output_ORS = os.path.join(report_directory, "tableau_output.gdb", "optimized_route_segments") - arcpy.Copy_management( - in_data=os.path.join(the_scenario.main_gdb, "optimized_route_segments"), - out_data=output_ORS, - data_type="FeatureClass") - # add field "record_id" - arcpy.AddField_management(output_ORS, "record_id", "TEXT") - - # field calculator; netsource + netsource_oid for unique field - arcpy.CalculateField_management(in_table=output_ORS, field="record_id", - expression='"{}_{} ".format(!OBJECTID!, !NET_SOURCE_NAME!)', - expression_type="PYTHON_9.3", code_block="") - - # add the scenario_name in the optimized_route_segments_fc (output_ORS) - arcpy.AddField_management(output_ORS, - "Scenario_Name", - "TEXT") - arcpy.CalculateField_management(in_table=output_ORS, field="Scenario_Name", - expression='"{}".format(the_scenario.scenario_name)', - expression_type="PYTHON_9.3", code_block="") - - # Create the zip file for writing compressed data - - logger.debug('creating archive') - gdb_filename = os.path.join(report_directory, "tableau_output.gdb") - zip_gdb_filename = gdb_filename + ".zip" - zf = zipfile.ZipFile(zip_gdb_filename, 'w', zipfile.ZIP_DEFLATED) - - zipgdb(gdb_filename, zf) - zf.close() - logger.debug('all done zipping') - - # now delete the unzipped gdb - arcpy.Delete_management(gdb_filename) - logger.debug("deleted unzipped {} gdb".format("tableau_output.gdb")) - - # copy the relative path tableau TWB file from the common data director to - # the timestamped tableau report directory - logger.debug("copying the twb file from common data to the timestamped tableau report folder.") - root_twb_location = os.path.join(the_scenario.common_data_folder, "tableau_dashboard.twb") - root_graphic_location = os.path.join(the_scenario.common_data_folder, "volpeTriskelion.gif") - root_config_parameters_graphic_location = os.path.join(the_scenario.common_data_folder, "parameters_icon.png") - scenario_twb_location = os.path.join(report_directory, "tableau_dashboard.twb") - scenario_graphic_location = os.path.join(report_directory, "volpeTriskelion.gif") - scenario_config_parameters_graphic_location = os.path.join(report_directory, "parameters_icon.png") - copy(root_twb_location, scenario_twb_location) - copy(root_graphic_location, scenario_graphic_location) - copy(root_config_parameters_graphic_location, scenario_config_parameters_graphic_location) - - # copy tableau report to the assets location - latest_generic_path = os.path.join(report_directory, "tableau_report.csv") - logger.debug("copying the latest tableau report csv file to the timestamped tableau report directory") - copy(report_file, latest_generic_path) - - # create packaged workbook for tableau reader compatibility - twbx_dashboard_filename = os.path.join(report_directory, "tableau_dashboard.twbx") - zipObj = zipfile.ZipFile(twbx_dashboard_filename, 'w', zipfile.ZIP_DEFLATED) - - # Add multiple files to the zip - # need to specify the arcname parameter to avoid the whole path to the file being added to the archive - zipObj.write(os.path.join(report_directory, "tableau_dashboard.twb"), "tableau_dashboard.twb") - zipObj.write(os.path.join(report_directory, "tableau_report.csv"), "tableau_report.csv") - zipObj.write(os.path.join(report_directory, "volpeTriskelion.gif"), "volpeTriskelion.gif") - zipObj.write(os.path.join(report_directory, "parameters_icon.png"), "parameters_icon.png") - zipObj.write(os.path.join(report_directory, "tableau_output.gdb.zip"), "tableau_output.gdb.zip") - - # close the Zip File - zipObj.close() - - # delete the other four files so its nice an clean. - os.remove(os.path.join(report_directory, "tableau_dashboard.twb")) - os.remove(os.path.join(report_directory, "tableau_report.csv")) - os.remove(os.path.join(report_directory, "volpeTriskelion.gif")) - os.remove(os.path.join(report_directory, "parameters_icon.png")) - os.remove(os.path.join(report_directory, "tableau_output.gdb.zip")) - - return report_directory - - -# ================================================================== - - -def generate_reports(the_scenario, logger): - logger.info("start: parse log operation for reports") - report_directory = os.path.join(the_scenario.scenario_run_directory, "Reports") - if not os.path.exists(report_directory): - os.makedirs(report_directory) - - filetype_list = ['s_', 'f_', 'f2', 'c_', 'c2', 'g_', 'g2', 'o', 'o1', 'o2', 'oc', 'oc1', 'oc2', 'oc3', 'os', 'p_'] - # init the dictionary to hold them by type. for the moment ignoring other types. - log_file_dict = {} - for x in filetype_list: - log_file_dict[x] = [] - - # get all of the log files matching the pattern - log_files = glob.glob(os.path.join(the_scenario.scenario_run_directory, "logs", "*_log_*_*_*_*-*-*.log")) - - # add log file name and date to dictionary. each entry in the array - # will be a tuple of (log_file_name, datetime object) - - for log_file in log_files: - - path_to, the_file_name = ntpath.split(log_file) - the_type = the_file_name[:2] - - # one letter switch 's_log_', 'd_log_', etc. - if the_file_name[5] == "_": - the_date = datetime.datetime.strptime(the_file_name[5:25], "_%Y_%m_%d_%H-%M-%S") - - # two letter switch 'c2_log_', 'g2_log_', etc. - elif the_file_name[6] == "_": - the_date = datetime.datetime.strptime(the_file_name[6:26], "_%Y_%m_%d_%H-%M-%S") - else: - logger.warning("The filename: {} is not supported in the logging".format(the_file_name)) - - if the_type in log_file_dict: - log_file_dict[the_type].append((the_file_name, the_date)) - - # sort each log type list by datetime so the most recent is first. - for x in filetype_list: - log_file_dict[x] = sorted(log_file_dict[x], key=lambda tup: tup[1], reverse=True) - - # create a list of log files to include in report by grabbing the latest version. - # these will be in order by log type (i.e. s, f, c, g, etc) - most_recent_log_file_set = [] - - if len(log_file_dict['s_']) > 0: - most_recent_log_file_set.append(log_file_dict['s_'][0]) - - if len(log_file_dict['f_']) > 0: - most_recent_log_file_set.append(log_file_dict['f_'][0]) - - if len(log_file_dict['c_']) > 0: - most_recent_log_file_set.append(log_file_dict['c_'][0]) - - if len(log_file_dict['g_']) > 0: - most_recent_log_file_set.append(log_file_dict['g_'][0]) - - if len(log_file_dict['oc']) == 0: - if len(log_file_dict['o']) > 0: - most_recent_log_file_set.append(log_file_dict['o'][0]) - - if len(log_file_dict['o1']) > 0: - most_recent_log_file_set.append(log_file_dict['o1'][0]) - - if len(log_file_dict['o2']) > 0: - most_recent_log_file_set.append(log_file_dict['o2'][0]) - - if len(log_file_dict['os']) > 0: - most_recent_log_file_set.append(log_file_dict['os'][0]) - - if len(log_file_dict['oc']) > 0: - most_recent_log_file_set.append(log_file_dict['oc'][0]) - - if len(log_file_dict['f2']) > 0: - most_recent_log_file_set.append(log_file_dict['f2'][0]) - - if len(log_file_dict['c2']) > 0: - most_recent_log_file_set.append(log_file_dict['c2'][0]) - - if len(log_file_dict['g2']) > 0: - most_recent_log_file_set.append(log_file_dict['g2'][0]) - - if len(log_file_dict['o']) > 0: - most_recent_log_file_set.append(log_file_dict['o'][0]) - - if len(log_file_dict['o1']) > 0: - most_recent_log_file_set.append(log_file_dict['o1'][0]) - - if len(log_file_dict['o2']) > 0: - most_recent_log_file_set.append(log_file_dict['o2'][0]) - - if len(log_file_dict['p_']) > 0: - most_recent_log_file_set.append(log_file_dict['p_'][0]) - - # figure out the last index of most_recent_log_file_set to include - # by looking at dates. if a subsequent step is seen to have an older - # log than a preceding step, no subsequent logs will be used. - # -------------------------------------------------------------------- - - last_index_to_include = 0 - - for i in range(1, len(most_recent_log_file_set)): - # print most_recent_log_file_set[i] - if i == 1: - last_index_to_include += 1 - elif i > 1: - if most_recent_log_file_set[i][1] > most_recent_log_file_set[i - 1][1]: - last_index_to_include += 1 - else: - break - - # print last_index_to_include - # -------------------------------------------------------- - - message_dict = { - 'RESULT': [], - 'CONFIG': [], - 'ERROR': [], - 'WARNING': [], - 'RUNTIME': [] - } - - for i in range(0, last_index_to_include + 1): - - in_file = os.path.join(the_scenario.scenario_run_directory, "logs", most_recent_log_file_set[i][0]) - - # s, p, b, r - record_src = most_recent_log_file_set[i][0][:2].upper() - - with open(in_file, 'r') as rf: - for line in rf: - recs = line.strip()[19:].split(' ', 1) - if recs[0] in message_dict: - if len(recs) > 1: # RE: Issue #182 - exceptions at the end of the log will cause this to fail. - message_dict[recs[0]].append((record_src, recs[1].strip())) - - # dump to file - # --------------- - timestamp = datetime.datetime.now() - report_file_name = 'report_' + timestamp.strftime("%Y_%m_%d_%H-%M-%S") + ".txt" - - report_file = os.path.join(report_directory, report_file_name) - with open(report_file, 'w') as wf: - - wf.write('SCENARIO\n') - wf.write('---------------------------------------------------------------------\n') - wf.write('Scenario Name\t:\t{}\n'.format(the_scenario.scenario_name)) - wf.write('Timestamp\t:\t{}\n'.format(timestamp.strftime("%Y-%m-%d %H:%M:%S"))) - wf.write('FTOT Version\t:\t{}\n'.format(FTOT_VERSION)) - - wf.write('\nRUNTIME\n') - wf.write('---------------------------------------------------------------------\n') - for x in message_dict['RUNTIME']: - wf.write('{}\t:\t{}\n'.format(x[0], x[1])) - - wf.write('\nRESULTS\n') - wf.write('---------------------------------------------------------------------\n') - for x in message_dict['RESULT']: - wf.write('{}\t:\t{}\n'.format(x[0], x[1])) - - wf.write('\nCONFIG\n') - wf.write('---------------------------------------------------------------------\n') - for x in message_dict['CONFIG']: - wf.write('{}\t:\t{}\n'.format(x[0], x[1])) - - if len(message_dict['ERROR']) > 0: - wf.write('\nERROR\n') - wf.write('---------------------------------------------------------------------\n') - for x in message_dict['ERROR']: - wf.write('{}\t:\t\t{}\n'.format(x[0], x[1])) - - if len(message_dict['WARNING']) > 0: - wf.write('\nWARNING\n') - wf.write('---------------------------------------------------------------------\n') - for x in message_dict['WARNING']: - wf.write('{}\t:\t\t{}\n'.format(x[0], x[1])) - - logger.info("Done Parse Log Operation") - logger.info("Report file location: {}".format(report_file)) - - # ------------------------------------------------------------- - - logger.info("start: Tableau results report") - report_file_name = 'tableau_report_' + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + "_" + str( - the_scenario.scenario_name).replace(" ", "_") + ".csv" - report_file_name = clean_file_name(report_file_name) - report_file = os.path.join(report_directory, report_file_name) - - with open(report_file, 'w') as wf: - wf.write('scenario_name, table_name, commodity, facility_name, measure, mode, value, units, notes\n') - import sqlite3 - - with sqlite3.connect(the_scenario.main_db) as db_con: - - # query the optimal scenario results table and report out the results - # ------------------------------------------------------------------------- - sql = "select * from optimal_scenario_results order by table_name, commodity, measure, mode;" - db_cur = db_con.execute(sql) - data = db_cur.fetchall() - - for row in data: - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, row[0], row[1], - row[2], row[3], row[4], row[5], row[6], row[7])) - - # Scenario Total Supply and Demand, and Available Processing Capacity - # note: processor input and outputs are based on facility size and reflect a processing capacity, - # not a conversion of the scenario feedstock supply - # ------------------------------------------------------------------------- - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc;""" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - for row in db_data: - facility_type = row[1] - io = row[2] - if facility_type == "raw_material_producer": - measure = "supply" - if facility_type == "processor" and io == 'o': - measure = "processing output capacity" - if facility_type == "processor" and io == 'i': - measure = "processing input capacity" - if facility_type == "ultimate_destination": - measure = "demand" - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, - "total_supply_demand_proc", - row[0], - "all_{}".format(row[1]), - measure, - io, - row[3], - row[4], - None)) - - # Scenario Stranded Supply, Demand, and Processing Capacity - # note: stranded supply refers to facilities that are ignored from the analysis.") - # ------------------------------------------------------------------------- - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units, f.ignore_facility - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility != 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units, f.ignore_facility - order by commodity_name, io asc;""" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - - for row in db_data: - facility_type = row[1] - io = row[2] - if facility_type == "raw_material_producer": - measure = "stranded supply" - if facility_type == "processor" and io == 'o': - measure = "stranded processing output capacity" - if facility_type == "processor" and io == 'i': - measure = "stranded processing input capacity" - if facility_type == "ultimate_destination": - measure = "stranded demand" - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, - "stranded_supply_demand_proc", - row[0], - "stranded_{}".format(row[1]), - measure, - io, - row[3], - row[4], - row[5])) # ignore_facility note - - # report out net quantities with ignored facilities removed from the query - # note: net supply, demand, and processing capacity ignores facilities not connected to the network - # ------------------------------------------------------------------------- - sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units - from facility_commodities fc - join commodities c on fc.commodity_id = c.commodity_id - join facilities f on f.facility_id = fc.facility_id - join facility_type_id fti on fti.facility_type_id = f.facility_type_id - where f.ignore_facility == 'false' - group by c.commodity_name, fc.io, fti.facility_type, fc.units - order by commodity_name, io desc;""" - db_cur = db_con.execute(sql) - db_data = db_cur.fetchall() - for row in db_data: - facility_type = row[1] - io = row[2] - if facility_type == "raw_material_producer": - measure = "net supply" - if facility_type == "processor" and io == 'o': - measure = "net processing output capacity" - if facility_type == "processor" and io == 'i': - measure = "net processing input capacity" - if facility_type == "ultimate_destination": - measure = "net demand" - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, - "net_supply_demand_proc", - row[0], - "net_{}".format(row[1]), - measure, - io, - row[3], - row[4], - None)) - - # REPORT OUT CONFIG FOR O2 STEP AND RUNTIMES FOR ALL STEPS - # Loop through the list of configurations records in the message_dict['config'] and ['runtime']. - # Note that this list is of the format: ['step', "xml_record, value"], and additional parsing is required. - step_to_export = 'O2' # the step we want to export - logger.info("output the configuration for step: {}".format(step_to_export)) - for config_record in message_dict['CONFIG']: - step = config_record[0] - if step != step_to_export: - continue - else: - # parse the xml record and value from the - xml_record_and_value = config_record[1] - xml_record = xml_record_and_value.split(":", 1)[0].replace("xml_", "") - value = xml_record_and_value.split(":", 1)[1].strip() # only split on the first colon to prevent paths from being split - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, "config", '', '', - xml_record, '', '', '', - value)) - - for x in message_dict['RUNTIME']: - # message_dict['RUNTIME'][0] - # ('S_', 's Step - Total Runtime (HMS): \t00:00:21') - step, runtime = x[1].split('\t') - wf.write("{}, {}, {}, {}, {}, {}, {}, {}, {}\n".format(the_scenario.scenario_name, "runtime", '', '', - step, '', '', '', - runtime)) - - logger.debug("finish: Tableau results report operation") - - latest_report_dir = prepare_tableau_assets(report_file, the_scenario, logger) - - logger.result("Tableau Report file location: {}".format(report_file)) +# --------------------------------------------------------------------------------------------------- +# Name: ftot_report +# +# Purpose: parses the most current log file for each step and assembles two reports. +# the first is a human readable report that groups log messages by runtime, result, config, and warnings. +# The second report is formatted for FTOT tableau dashboard as a simple CSV and comes from a few tables in the db. +# +# --------------------------------------------------------------------------------------------------- + +import os +import datetime +import glob +import ntpath +import zipfile +import sqlite3 +import csv +from ftot_supporting_gis import zipgdb +from ftot_supporting import clean_file_name +from shutil import copy + +from ftot import FTOT_VERSION + +TIMESTAMP = datetime.datetime.now() + +# ================================================================== + + +def prepare_tableau_assets(timestamp_directory, report_file, the_scenario, logger): + logger.info("start: prepare_tableau_assets") + + # make tableau_output.gdb file + # ----------------------------------- + logger.debug("start: create tableau gdb") + import arcpy + arcpy.CreateFileGDB_management(out_folder_path=timestamp_directory, + out_name="tableau_output", out_version="CURRENT") + + # add Dataset field (raw_material_producers, processors, ultimate_destinations) + arcpy.AddField_management(the_scenario.rmp_fc, "Dataset", "TEXT") + arcpy.CalculateField_management(in_table=the_scenario.rmp_fc, field="Dataset", + expression='"raw_material_producers"', + expression_type="PYTHON_9.3", code_block="") + arcpy.AddField_management(the_scenario.processors_fc, "Dataset", "TEXT") + arcpy.CalculateField_management(in_table=the_scenario.processors_fc, field="Dataset", + expression='"processors"', + expression_type="PYTHON_9.3", code_block="") + arcpy.AddField_management(the_scenario.destinations_fc, "Dataset", "TEXT") + arcpy.CalculateField_management(in_table=the_scenario.destinations_fc, field="Dataset", + expression='"ultimate_destinations"', + expression_type="PYTHON_9.3", code_block="") + + # merge facility type FCs + facilities_merge_fc = os.path.join(timestamp_directory, "tableau_output.gdb", "facilities_merge") + arcpy.Merge_management([the_scenario.destinations_fc, the_scenario.rmp_fc, the_scenario.processors_fc], + facilities_merge_fc) + + # add the scenario_name in the facilities_merge + arcpy.AddField_management(facilities_merge_fc, + "Scenario_Name", + "TEXT") + arcpy.CalculateField_management(in_table=facilities_merge_fc, field="Scenario_Name", + expression='"{}".format(the_scenario.scenario_name)', + expression_type="PYTHON_9.3", code_block="") + + + + # copy optimized_route_segments_disolved (aka: ORSD) + output_ORSD = os.path.join(timestamp_directory, "tableau_output.gdb", "optimized_route_segments_dissolved") + arcpy.Copy_management( + in_data=os.path.join(the_scenario.main_gdb, "optimized_route_segments_dissolved"), + out_data=output_ORSD, + data_type="FeatureClass") + # add field "record_id" + arcpy.AddField_management(output_ORSD, "record_id", "TEXT") + + # field calculator; netsource + netsource_oid for unique field + arcpy.CalculateField_management(in_table=output_ORSD, field="record_id", + expression='"{}_{} ".format(!OBJECTID!, !NET_SOURCE_NAME!)', + expression_type="PYTHON_9.3", code_block="") + + # add the scenario_name in the optimized_route_segments_dissolved_fc (output_ORSD) + arcpy.AddField_management(output_ORSD, + "Scenario_Name", + "TEXT") + arcpy.CalculateField_management(in_table=output_ORSD, field="Scenario_Name", + expression='"{}".format(the_scenario.scenario_name)', + expression_type="PYTHON_9.3", code_block="") + + # copy optimized_route_segments (ORS) + # this contains commodity info at the link level + output_ORS = os.path.join(timestamp_directory, "tableau_output.gdb", "optimized_route_segments") + arcpy.Copy_management( + in_data=os.path.join(the_scenario.main_gdb, "optimized_route_segments"), + out_data=output_ORS, + data_type="FeatureClass") + # add field "record_id" + arcpy.AddField_management(output_ORS, "record_id", "TEXT") + + # field calculator; netsource + netsource_oid for unique field + arcpy.CalculateField_management(in_table=output_ORS, field="record_id", + expression='"{}_{} ".format(!OBJECTID!, !NET_SOURCE_NAME!)', + expression_type="PYTHON_9.3", code_block="") + + # add the scenario_name in the optimized_route_segments_fc (output_ORS) + arcpy.AddField_management(output_ORS, + "Scenario_Name", + "TEXT") + arcpy.CalculateField_management(in_table=output_ORS, field="Scenario_Name", + expression='"{}".format(the_scenario.scenario_name)', + expression_type="PYTHON_9.3", code_block="") + + # Create the zip file for writing compressed data + + logger.debug('creating archive') + gdb_filename = os.path.join(timestamp_directory, "tableau_output.gdb") + zip_gdb_filename = gdb_filename + ".zip" + zf = zipfile.ZipFile(zip_gdb_filename, 'w', zipfile.ZIP_DEFLATED) + + zipgdb(gdb_filename, zf) + zf.close() + logger.debug('all done zipping') + + # now delete the unzipped gdb + arcpy.Delete_management(gdb_filename) + logger.debug("deleted unzipped {} gdb".format("tableau_output.gdb")) + + # copy the relative path tableau TWB file from the common data director to + # the timestamped tableau report directory + logger.debug("copying the twb file from common data to the timestamped tableau report folder.") + root_twb_location = os.path.join(the_scenario.common_data_folder, "tableau_dashboard.twb") + root_graphic_location = os.path.join(the_scenario.common_data_folder, "volpeTriskelion.gif") + root_config_parameters_graphic_location = os.path.join(the_scenario.common_data_folder, "parameters_icon.png") + scenario_twb_location = os.path.join(timestamp_directory, "tableau_dashboard.twb") + scenario_graphic_location = os.path.join(timestamp_directory, "volpeTriskelion.gif") + scenario_config_parameters_graphic_location = os.path.join(timestamp_directory, "parameters_icon.png") + copy(root_twb_location, scenario_twb_location) + copy(root_graphic_location, scenario_graphic_location) + copy(root_config_parameters_graphic_location, scenario_config_parameters_graphic_location) + + # copy tableau report to the assets location + latest_generic_path = os.path.join(timestamp_directory, "tableau_report.csv") + logger.debug("copying the latest tableau report csv file to the timestamped tableau report directory") + copy(report_file, latest_generic_path) + + # create packaged workbook for tableau reader compatibility + twbx_dashboard_filename = os.path.join(timestamp_directory, "tableau_dashboard.twbx") + zipObj = zipfile.ZipFile(twbx_dashboard_filename, 'w', zipfile.ZIP_DEFLATED) + + # Add multiple files to the zip + # need to specify the arcname parameter to avoid the whole path to the file being added to the archive + zipObj.write(os.path.join(timestamp_directory, "tableau_dashboard.twb"), "tableau_dashboard.twb") + zipObj.write(os.path.join(timestamp_directory, "tableau_report.csv"), "tableau_report.csv") + zipObj.write(os.path.join(timestamp_directory, "volpeTriskelion.gif"), "volpeTriskelion.gif") + zipObj.write(os.path.join(timestamp_directory, "parameters_icon.png"), "parameters_icon.png") + zipObj.write(os.path.join(timestamp_directory, "tableau_output.gdb.zip"), "tableau_output.gdb.zip") + + # close the Zip File + zipObj.close() + + # delete the other four files so its nice an clean. + os.remove(os.path.join(timestamp_directory, "tableau_dashboard.twb")) + os.remove(os.path.join(timestamp_directory, "tableau_report.csv")) + os.remove(os.path.join(timestamp_directory, "volpeTriskelion.gif")) + os.remove(os.path.join(timestamp_directory, "parameters_icon.png")) + os.remove(os.path.join(timestamp_directory, "tableau_output.gdb.zip")) + + +# ============================================================================================== + +def generate_edges_from_routes_summary(timestamp_directory, the_scenario, logger): + + logger.info("start: generate_edges_from_routes_summary") + report_file_name = 'optimal_routes_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + ".csv" + report_file_name = clean_file_name(report_file_name) + report_file = os.path.join(timestamp_directory, report_file_name) + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + summary_route_data = main_db_con.execute("""select rr.route_id, f1.facility_name as from_facility, + f2.facility_name as to_facility, rr.phase_of_matter, rr.dollar_cost, rr.cost, rr.miles + FROM route_reference rr + join facilities f1 on rr.from_facility_id = f1.facility_id + join facilities f2 on rr.to_facility_id = f2.facility_id; """) + + # Print route data to file in debug folder + with open(report_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(['route_id','from_facility','to_facility','phase','dollar_cost','routing_cost','miles']) + writer.writerows(summary_route_data) + + +# ============================================================================================== + +def generate_artificial_link_summary(timestamp_directory, the_scenario, logger): + + logger.info("start: generate_artificial_link_summary") + report_file_name = 'artificial_links_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + ".csv" + report_file_name = clean_file_name(report_file_name) + report_file = os.path.join(timestamp_directory, report_file_name) + + # query the facility and network tables for artificial links and report out the results + with sqlite3.connect(the_scenario.main_db) as db_con: + sql = """select fac.facility_name, fti.facility_type, ne.mode_source, round(ne.miles, 3) as miles + from facilities fac + left join facility_type_id fti on fac.facility_type_id = fti.facility_type_id + left join networkx_nodes nn on fac.location_id = nn.location_id + left join networkx_edges ne on nn.node_id = ne.from_node_id + where nn.location_1 like '%OUT%' + and ne.artificial = 1 + ;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + + artificial_links = {} + for row in db_data: + facility_name = row[0] + facility_type = row[1] + mode_source = row[2] + link_length = row[3] + + if facility_name not in artificial_links: + # add new facility to dictionary and start list of artificial links by mode + artificial_links[facility_name] = {'fac_type': facility_type, 'link_lengths': {}} + + if mode_source not in artificial_links[facility_name]['link_lengths']: + artificial_links[facility_name]['link_lengths'][mode_source] = link_length + else: + # there should only be one artificial link for a facility for each mode + error = "Multiple artificial links should not be found for a single facility for a particular mode." + logger.error(error) + raise Exception(error) + + # create structure for artificial link csv table + output_table = {'facility_name': [], 'facility_type': []} + for permitted_mode in the_scenario.permittedModes: + output_table[permitted_mode] = [] + + # iterate through every facility, add a row to csv table + for k in artificial_links: + output_table['facility_name'].append(k) + output_table['facility_type'].append(artificial_links[k]['fac_type']) + for permitted_mode in the_scenario.permittedModes: + if permitted_mode in artificial_links[k]['link_lengths']: + output_table[permitted_mode].append(artificial_links[k]['link_lengths'][permitted_mode]) + else: + output_table[permitted_mode].append('NA') + + # print artificial link data for each facility to file in debug folder + with open(report_file, 'w', newline='') as f: + writer = csv.writer(f) + output_fields = ['facility_name', 'facility_type'] + the_scenario.permittedModes + writer.writerow(output_fields) + writer.writerows(zip(*[output_table[key] for key in output_fields])) + + logger.debug("finish: generate_artificial_link_summary") + +# ============================================================================================== + +def generate_detailed_emissions_summary(timestamp_directory, the_scenario, logger): + + logger.info("start: generate_detailed_emissions_summary") + report_file_name = 'detailed_emissions_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + ".csv" + report_file_name = clean_file_name(report_file_name) + report_file = os.path.join(timestamp_directory, report_file_name) + + # query the emissions tables and report out the results + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + emissions_data = main_db_con.execute("select * from detailed_emissions;") + emissions_data = emissions_data.fetchall() + + # print emissions data to new report file + with open(report_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(['commodity','mode','pollutant','value','units']) + writer.writerows(emissions_data) + + logger.debug("finish: generate_detailed_emissions_summary") + +# ================================================================== + + +def generate_reports(the_scenario, logger): + logger.info("start: parse log operation for reports") + + # make overall Reports directory + report_directory = os.path.join(the_scenario.scenario_run_directory, "Reports") + if not os.path.exists(report_directory): + os.makedirs(report_directory) + + # make subdirectory for individual run + timestamp_folder_name = 'reports_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + timestamp_directory = os.path.join(the_scenario.scenario_run_directory, "Reports", timestamp_folder_name) + if not os.path.exists(timestamp_directory): + os.makedirs(timestamp_directory) + + filetype_list = ['s_', 'f_', 'f2', 'c_', 'c2', 'g_', 'g2', 'o', 'o1', 'o2', 'oc', 'oc1', 'oc2', 'oc3', 'os', 'p_'] + # init the dictionary to hold them by type. for the moment ignoring other types. + log_file_dict = {} + for x in filetype_list: + log_file_dict[x] = [] + + # get all of the log files matching the pattern + log_files = glob.glob(os.path.join(the_scenario.scenario_run_directory, "logs", "*_log_*_*_*_*-*-*.log")) + + # add log file name and date to dictionary. each entry in the array + # will be a tuple of (log_file_name, datetime object) + + for log_file in log_files: + + path_to, the_file_name = ntpath.split(log_file) + the_type = the_file_name[:2] + + # one letter switch 's_log_', 'd_log_', etc. + if the_file_name[5] == "_": + the_date = datetime.datetime.strptime(the_file_name[5:25], "_%Y_%m_%d_%H-%M-%S") + + # two letter switch 'c2_log_', 'g2_log_', etc. + elif the_file_name[6] == "_": + the_date = datetime.datetime.strptime(the_file_name[6:26], "_%Y_%m_%d_%H-%M-%S") + else: + logger.warning("The filename: {} is not supported in the logging".format(the_file_name)) + + if the_type in log_file_dict: + log_file_dict[the_type].append((the_file_name, the_date)) + + # sort each log type list by datetime so the most recent is first. + for x in filetype_list: + log_file_dict[x] = sorted(log_file_dict[x], key=lambda tup: tup[1], reverse=True) + + # create a list of log files to include in report by grabbing the latest version. + # these will be in order by log type (i.e. s, f, c, g, etc) + most_recent_log_file_set = [] + + if len(log_file_dict['s_']) > 0: + most_recent_log_file_set.append(log_file_dict['s_'][0]) + + if len(log_file_dict['f_']) > 0: + most_recent_log_file_set.append(log_file_dict['f_'][0]) + + if len(log_file_dict['c_']) > 0: + most_recent_log_file_set.append(log_file_dict['c_'][0]) + + if len(log_file_dict['g_']) > 0: + most_recent_log_file_set.append(log_file_dict['g_'][0]) + + if len(log_file_dict['oc']) == 0: + if len(log_file_dict['o']) > 0: + most_recent_log_file_set.append(log_file_dict['o'][0]) + + if len(log_file_dict['o1']) > 0: + most_recent_log_file_set.append(log_file_dict['o1'][0]) + + if len(log_file_dict['o2']) > 0: + most_recent_log_file_set.append(log_file_dict['o2'][0]) + + if len(log_file_dict['os']) > 0: + most_recent_log_file_set.append(log_file_dict['os'][0]) + + if len(log_file_dict['oc']) > 0: + most_recent_log_file_set.append(log_file_dict['oc'][0]) + + if len(log_file_dict['f2']) > 0: + most_recent_log_file_set.append(log_file_dict['f2'][0]) + + if len(log_file_dict['c2']) > 0: + most_recent_log_file_set.append(log_file_dict['c2'][0]) + + if len(log_file_dict['g2']) > 0: + most_recent_log_file_set.append(log_file_dict['g2'][0]) + + if len(log_file_dict['o']) > 0: + most_recent_log_file_set.append(log_file_dict['o'][0]) + + if len(log_file_dict['o1']) > 0: + most_recent_log_file_set.append(log_file_dict['o1'][0]) + + if len(log_file_dict['o2']) > 0: + most_recent_log_file_set.append(log_file_dict['o2'][0]) + + if len(log_file_dict['p_']) > 0: + most_recent_log_file_set.append(log_file_dict['p_'][0]) + + # figure out the last index of most_recent_log_file_set to include + # by looking at dates. if a subsequent step is seen to have an older + # log than a preceding step, no subsequent logs will be used. + # -------------------------------------------------------------------- + + last_index_to_include = 0 + + for i in range(1, len(most_recent_log_file_set)): + # print most_recent_log_file_set[i] + if i == 1: + last_index_to_include += 1 + elif i > 1: + if most_recent_log_file_set[i][1] > most_recent_log_file_set[i - 1][1]: + last_index_to_include += 1 + else: + break + + # print last_index_to_include + # -------------------------------------------------------- + + message_dict = { + 'RESULT': [], + 'CONFIG': [], + 'ERROR': [], + 'WARNING': [], + 'RUNTIME': [] + } + + for i in range(0, last_index_to_include + 1): + + in_file = os.path.join(the_scenario.scenario_run_directory, "logs", most_recent_log_file_set[i][0]) + + # s, p, b, r + record_src = most_recent_log_file_set[i][0][:2].upper() + + with open(in_file, 'r') as rf: + for line in rf: + recs = line.strip()[19:].split(' ', 1) + if recs[0] in message_dict: + if len(recs) > 1: # RE: Issue #182 - exceptions at the end of the log will cause this to fail. + message_dict[recs[0]].append((record_src, recs[1].strip())) + + # dump to file + # --------------- + + report_file_name = 'report_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + ".txt" + + report_file = os.path.join(timestamp_directory, report_file_name) + with open(report_file, 'w') as wf: + + wf.write('SCENARIO\n') + wf.write('---------------------------------------------------------------------\n') + wf.write('Scenario Name\t:\t{}\n'.format(the_scenario.scenario_name)) + wf.write('Timestamp\t:\t{}\n'.format(TIMESTAMP.strftime("%Y-%m-%d %H:%M:%S"))) + wf.write('FTOT Version\t:\t{}\n'.format(FTOT_VERSION)) + + wf.write('\nRUNTIME\n') + wf.write('---------------------------------------------------------------------\n') + for x in message_dict['RUNTIME']: + wf.write('{}\t:\t{}\n'.format(x[0], x[1])) + + wf.write('\nRESULTS\n') + wf.write('---------------------------------------------------------------------\n') + for x in message_dict['RESULT']: + wf.write('{}\t:\t{}\n'.format(x[0], x[1])) + + wf.write('\nCONFIG\n') + wf.write('---------------------------------------------------------------------\n') + for x in message_dict['CONFIG']: + wf.write('{}\t:\t{}\n'.format(x[0], x[1])) + + if len(message_dict['ERROR']) > 0: + wf.write('\nERROR\n') + wf.write('---------------------------------------------------------------------\n') + for x in message_dict['ERROR']: + wf.write('{}\t:\t\t{}\n'.format(x[0], x[1])) + + if len(message_dict['WARNING']) > 0: + wf.write('\nWARNING\n') + wf.write('---------------------------------------------------------------------\n') + for x in message_dict['WARNING']: + wf.write('{}\t:\t\t{}\n'.format(x[0], x[1])) + + logger.info("Done Parse Log Operation") + + # ------------------------------------------------------------- + + logger.info("start: main results report") + report_file_name = 'report_' + TIMESTAMP.strftime("%Y_%m_%d_%H-%M-%S") + ".csv" + report_file_name = clean_file_name(report_file_name) + report_file = os.path.join(timestamp_directory, report_file_name) + + with open(report_file, 'w', newline='') as wf: + writer = csv.writer(wf) + writer.writerow(['scenario_name', 'table_name', 'commodity', 'facility_name', 'measure', 'mode', 'value', 'units', 'notes']) + + import sqlite3 + with sqlite3.connect(the_scenario.main_db) as db_con: + + # query the optimal scenario results table and report out the results + # ------------------------------------------------------------------------- + sql = "select * from optimal_scenario_results order by table_name, commodity, measure, mode;" + db_cur = db_con.execute(sql) + data = db_cur.fetchall() + + for row in data: + writer.writerow([the_scenario.scenario_name, row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7]]) + # Scenario Total Supply and Demand, and Available Processing Capacity + # note: processor input and outputs are based on facility size and reflect a processing capacity, + # not a conversion of the scenario feedstock supply + # ------------------------------------------------------------------------- + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + for row in db_data: + facility_type = row[1] + io = row[2] + if facility_type == "raw_material_producer": + measure = "supply" + if facility_type == "processor" and io == 'o': + measure = "processing output capacity" + if facility_type == "processor" and io == 'i': + measure = "processing input capacity" + if facility_type == "ultimate_destination": + measure = "demand" + writer.writerow([the_scenario.scenario_name, "total_supply_demand_proc", row[0], "all_{}".format(row[1]), measure, io, row[3], row[4], None]) + + # Scenario Stranded Supply, Demand, and Processing Capacity + # note: stranded supply refers to facilities that are ignored from the analysis.") + # ------------------------------------------------------------------------- + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units, f.ignore_facility + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility != 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units, f.ignore_facility + order by commodity_name, io asc;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + + for row in db_data: + facility_type = row[1] + io = row[2] + if facility_type == "raw_material_producer": + measure = "stranded supply" + if facility_type == "processor" and io == 'o': + measure = "stranded processing output capacity" + if facility_type == "processor" and io == 'i': + measure = "stranded processing input capacity" + if facility_type == "ultimate_destination": + measure = "stranded demand" + writer.writerow([the_scenario.scenario_name, "stranded_supply_demand_proc", row[0], "stranded_{}".format(row[1]), measure, io, row[3], row[4], row[5]]) # ignore_facility note + + # report out net quantities with ignored facilities removed from the query + # note: net supply, demand, and processing capacity ignores facilities not connected to the network + # ------------------------------------------------------------------------- + sql = """ select c.commodity_name, fti.facility_type, io, sum(fc.quantity), fc.units + from facility_commodities fc + join commodities c on fc.commodity_id = c.commodity_id + join facilities f on f.facility_id = fc.facility_id + join facility_type_id fti on fti.facility_type_id = f.facility_type_id + where f.ignore_facility == 'false' + group by c.commodity_name, fc.io, fti.facility_type, fc.units + order by commodity_name, io desc;""" + db_cur = db_con.execute(sql) + db_data = db_cur.fetchall() + for row in db_data: + facility_type = row[1] + io = row[2] + if facility_type == "raw_material_producer": + measure = "net supply" + if facility_type == "processor" and io == 'o': + measure = "net processing output capacity" + if facility_type == "processor" and io == 'i': + measure = "net processing input capacity" + if facility_type == "ultimate_destination": + measure = "net demand" + writer.writerow([the_scenario.scenario_name, "net_supply_demand_proc", row[0], "net_{}".format(row[1]), measure, io, row[3], row[4], None]) + + # REPORT OUT CONFIG FOR O2 STEP AND RUNTIMES FOR ALL STEPS + # Loop through the list of configurations records in the message_dict['config'] and ['runtime']. + # Note that this list is of the format: ['step', "xml_record, value"], and additional parsing is required. + step_to_export = 'O2' # the step we want to export + logger.info("output the configuration for step: {}".format(step_to_export)) + for config_record in message_dict['CONFIG']: + step = config_record[0] + if step != step_to_export: + continue + else: + # parse the xml record and value from the + xml_record_and_value = config_record[1] + xml_record = xml_record_and_value.split(":", 1)[0].replace("xml_", "") + value = xml_record_and_value.split(":", 1)[1].strip() # only split on the first colon to prevent paths from being split + writer.writerow([the_scenario.scenario_name, "config", '', '', xml_record, '', '', '', value]) + + for x in message_dict['RUNTIME']: + # message_dict['RUNTIME'][0] + # ('S_', 's Step - Total Runtime (HMS): \t00:00:21') + step, runtime = x[1].split('\t') + writer.writerow([the_scenario.scenario_name, "runtime", '', '', step, '', '', '', runtime]) + + logger.debug("finish: main results report operation") + + prepare_tableau_assets(timestamp_directory, report_file, the_scenario, logger) + + # ------------------------------------------------------------- + + # artificial link summary + generate_artificial_link_summary(timestamp_directory, the_scenario, logger) + + # routes summary + if the_scenario.ndrOn: + generate_edges_from_routes_summary(timestamp_directory, the_scenario, logger) + + # emissions summary + if the_scenario.detailed_emissions: + generate_detailed_emissions_summary(timestamp_directory, the_scenario, logger) + + logger.result("Reports located here: {}".format(timestamp_directory)) diff --git a/program/ftot_routing.py b/program/ftot_routing.py index b298bc3..4c37976 100644 --- a/program/ftot_routing.py +++ b/program/ftot_routing.py @@ -1,1171 +1,1171 @@ -# ------------------------------------------------------------------------------ -# ftot_routing.py -# Purpose: the purpose of this module is to clean up, -# create the locations_fc, -# hook locations into the network, -# ignore locations not connected to the network, -# export capacity information to the main.db, -# export the assets from GIS export_fcs_from_main_gdb - -# Revised: 1/15/19 - MNP -# ------------------------------------------------------------------------------ - -import os -import arcpy -import sqlite3 -import ftot_supporting_gis -from ftot import Q_ - -LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') - - -# ======================================================================== - - -def connectivity(the_scenario, logger): - - checks_and_cleanup(the_scenario, logger) - - # create the locations_fc - create_locations_fc(the_scenario, logger) - - # use MBG to subset the road network to a buffer around the locations FC - minimum_bounding_geometry(the_scenario, logger) - - # hook locations into the network - hook_locations_into_network(the_scenario, logger) - - # ignore locations not connected to the network - ignore_locations_not_connected_to_network(the_scenario, logger) - - # report out material missing after connecting to the network - from ftot_facilities import db_report_commodity_potentials - db_report_commodity_potentials(the_scenario, logger) - - # export capacity information to the main.db - cache_capacity_information(the_scenario, logger) - - -# ========================================================================= - - -def checks_and_cleanup(the_scenario, logger): - logger.info("start: checks_and_cleanup") - - scenario_gdb = the_scenario.main_gdb - if not os.path.exists(scenario_gdb): - error = "can't find scenario gdb {}".format(scenario_gdb) - raise IOError(error) - - # check for scenario DB - # --------------------------------- - scenario_db = the_scenario.main_db - if not arcpy.Exists(scenario_db): - raise Exception("scenario_db not found {} ".format(scenario_db)) - - logger.debug("finish: checks_and_cleanup") - - -# ============================================================================== - - -def create_locations_fc(the_scenario, logger): - logger.info("start: create_locations_fc") - co_location_offet = 0.1 - logger.debug("co-location offset is necessary to prevent the locations from being treated as intermodal " - "facilities.") - logger.debug("collocation off-set: {} meters".format(co_location_offet)) - - locations_fc = the_scenario.locations_fc - - # delete the old location FC before we create it - from ftot_facilities import gis_clear_feature_class - gis_clear_feature_class(locations_fc, logger) - - # create the feature class - arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "locations", "POINT", "#", "DISABLED", "DISABLED", - ftot_supporting_gis.LCC_PROJ) - - # add the location_id field - arcpy.AddField_management(locations_fc, "location_id", "TEXT") - - # add the location_id_name field - arcpy.AddField_management(locations_fc, "location_id_name", "TEXT") - - # add the connects_road field - arcpy.AddField_management(locations_fc, "connects_road", "SHORT") - - # add the connects_rail field - arcpy.AddField_management(locations_fc, "connects_rail", "SHORT") - - # add the connects_water field - arcpy.AddField_management(locations_fc, "connects_water", "SHORT") - - # add the connects_pipeline_prod field - arcpy.AddField_management(locations_fc, "connects_pipeline_prod_trf_rts", "SHORT") - - # add the connects_pipeline_crude field - arcpy.AddField_management(locations_fc, "connects_pipeline_crude_trf_rts", "SHORT") - - # add the ignore field - arcpy.AddField_management(locations_fc, "ignore", "SHORT") - - # start an edit session for the insert cursor - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - # create insert cursor - with arcpy.da.InsertCursor(locations_fc, ["location_id_name", "location_id", "SHAPE@"]) as insert_cursor: - - # loop through DB and populate the fc - with sqlite3.connect(the_scenario.main_db) as db_con: - - sql = "select * from locations;" - db_cur = db_con.execute(sql) - for row in db_cur: - location_id = row[0] - - # create a point for each location "out" - location_point = arcpy.Point() - location_point.X = row[1] + co_location_offet - location_point.Y = row[2] + co_location_offet - location_point_geom = arcpy.PointGeometry(location_point, LCC_PROJ) - - insert_cursor.insertRow([str(location_id) + "_OUT", location_id, location_point_geom]) - - # create a point for each location "in" - location_point = arcpy.Point() - location_point.X = row[1] - co_location_offet - location_point.Y = row[2] - co_location_offet - location_point_geom = arcpy.PointGeometry(location_point, LCC_PROJ) - - insert_cursor.insertRow([str(location_id) + "_IN", location_id, location_point_geom]) - - edit.stopOperation() - edit.stopEditing(True) - - logger.debug("finish: create_locations_fc") - - -# ============================================================================== - - -def get_xy_location_id_dict(the_scenario, logger): - logger.debug("start: get_xy_location_id_dict") - - with sqlite3.connect(the_scenario.main_db) as db_con: - - sql = "select location_id, shape_x, shape_y from locations;" - db_cur = db_con.execute(sql) - - xy_location_id_dict = {} - for row in db_cur: - location_id = row[0] - shape_x = row[1] - shape_y = row[2] - xy_location_id_dict[location_id] = "[{}, {}]".format(shape_x, shape_y) - - logger.debug("finish: get_xy_location_id_dict") - - return xy_location_id_dict - - -# ============================================================================== - - -def get_location_id_name_dict(the_scenario, logger): - logger.debug("start: get_location_id_name_dict") - - location_id_name_dict = {} - - with arcpy.da.SearchCursor(the_scenario.locations_fc, ["location_id_name", "OBJECTID"]) as scursor: - - for row in scursor: - location_id_name = row[0] - objectid = row[1] - # shape = row[2] - location_id_name_dict[objectid] = location_id_name - - logger.debug("finish: get_location_id_name_dict") - - return location_id_name_dict - - -# =============================================================================== - - -def delete_old_artificial_link(the_scenario, logger): - logger.debug("start: delete_old_artificial_link") - for mode in ["road", "rail", "water", "pipeline_crude_trf_rts", "pipeline_prod_trf_rts"]: - edit = arcpy.da.Editor(the_scenario.main_gdb) - edit.startEditing(False, False) - edit.startOperation() - - with arcpy.da.UpdateCursor(os.path.join(the_scenario.main_gdb, 'network', mode), ["artificial"], - where_clause="Artificial = 1") as cursor: - for row in cursor: - cursor.deleteRow() - - edit.stopOperation() - edit.stopEditing(True) - logger.debug("finish: delete_old_artificial_link") - - -# =============================================================================== - - -def cut_lines(line_list, point_list, split_lines): - for line in line_list: - is_cut = "Not Cut" - if line.length > 0.0: # Make sure it's not an empty geometry. - for point in point_list: - # Even "coincident" points can show up as spatially non-coincident in their - # floating-point XY values, so we set up a tolerance. - if line.distanceTo(point) < 1.0: - # To ensure coincidence, snap the point to the line before proceeding. - snap_point = line.snapToLine(point).firstPoint - # Make sure the point isn't on a line endpoint, otherwise cutting will produce - # an empty geometry. - if not (snap_point.equals(line.lastPoint) and snap_point.equals(line.firstPoint)): - # Cut the line. Try it a few different ways to try increase the likelihood it will actually cut - cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( - [arcpy.Point(snap_point.X + 10.0, snap_point.Y + 10.0), - arcpy.Point(snap_point.X - 10.0, snap_point.Y - 10.0)]), LCC_PROJ)) - if cut_line_1.length == 0 or cut_line_2.length == 0: - cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( - [arcpy.Point(snap_point.X - 10.0, snap_point.Y + 10.0), - arcpy.Point(snap_point.X + 10.0, snap_point.Y - 10.0)]), LCC_PROJ)) - if cut_line_1.length == 0 or cut_line_2.length == 0: - cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( - [arcpy.Point(snap_point.X + 10.0, snap_point.Y), - arcpy.Point(snap_point.X - 10.0, snap_point.Y)]), LCC_PROJ)) - if cut_line_1.length == 0 or cut_line_2.length == 0: - cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( - [arcpy.Point(snap_point.X, snap_point.Y + 10.0), - arcpy.Point(snap_point.X, snap_point.Y - 10.0)]), LCC_PROJ)) - # Make sure both descendents have non-zero geometry. - if cut_line_1.length > 0.0 and cut_line_2.length > 0.0: - # Feed the cut lines back into the "line" list as candidates to be cut again. - line_list.append(cut_line_1) - line_list.append(cut_line_2) - line_list.remove(line) - # The cut loop will only exit when all lines cannot be cut smaller without producing - # zero-length geometries - is_cut = "Cut" - # break the loop because we've cut a line into two now and need to start over. - break - point_list.remove(point) - - if is_cut == "Not Cut" and len(point_list) == 0: - split_lines.append(line) - line_list.remove(line) - - if len(line_list) == 0 and len(point_list) == 0: - continue_iteration = 'done' - else: - continue_iteration = 'continue running' - return line_list, point_list, split_lines, continue_iteration - -# =============================================================================== - - -def hook_locations_into_network(the_scenario, logger): - - # Add artificial links from the locations feature class into the network - # ----------------------------------------------------------------------- - logger.info("start: hook_location_into_network") - - scenario_gdb = the_scenario.main_gdb - if not os.path.exists(scenario_gdb): - error = "can't find scenario gdb {}".format(scenario_gdb) - raise IOError(error) - - # LINKS TO/FROM LOCATIONS - # --------------------------- - road_max_artificial_link_distance_miles = str(the_scenario.road_max_artificial_link_dist.magnitude) + " Miles" - rail_max_artificial_link_distance_miles = str(the_scenario.rail_max_artificial_link_dist.magnitude) + " Miles" - water_max_artificial_link_distance_miles = str(the_scenario.water_max_artificial_link_dist.magnitude) + " Miles" - pipeline_crude_max_artificial_link_distance_miles = str(the_scenario.pipeline_crude_max_artificial_link_dist.magnitude) + " Miles" - pipeline_prod_max_artificial_link_distance_miles = str(the_scenario.pipeline_prod_max_artificial_link_dist.magnitude) + " Miles" - - # cleanup any old artificial links - delete_old_artificial_link(the_scenario, logger) - - # LINKS TO/FROM LOCATIONS - # --------------------------- - locations_fc = the_scenario.locations_fc - - arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_road", "SHORT") - arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_road", 0, "PYTHON_9.3") - - arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_rail", "SHORT") - arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_rail", 0, "PYTHON_9.3") - - arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_water", "SHORT") - arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_water", 0, "PYTHON_9.3") - - arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_prod_trf_rts", "SHORT") - arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_prod_trf_rts", 0, - "PYTHON_9.3") - - arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_crude_trf_rts", "SHORT") - arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_crude_trf_rts", 0, - "PYTHON_9.3") - - # check for permitted modes before creating artificial links - from ftot_networkx import check_permitted_modes - check_permitted_modes(the_scenario, logger) - - if 'road' in the_scenario.permittedModes: - locations_add_links(logger, the_scenario, "road", road_max_artificial_link_distance_miles) - if 'rail' in the_scenario.permittedModes: - locations_add_links(logger, the_scenario, "rail", rail_max_artificial_link_distance_miles) - if 'water' in the_scenario.permittedModes: - locations_add_links(logger, the_scenario, "water", water_max_artificial_link_distance_miles) - if 'pipeline_crude_trf_rts' in the_scenario.permittedModes: - locations_add_links(logger, the_scenario, "pipeline_crude_trf_rts", - pipeline_crude_max_artificial_link_distance_miles) - if 'pipeline_prod_trf_rts' in the_scenario.permittedModes: - locations_add_links(logger, the_scenario, "pipeline_prod_trf_rts", - pipeline_prod_max_artificial_link_distance_miles) - - # ADD THE SOURCE AND SOURCE_OID FIELDS SO WE CAN MAP THE LINKS IN THE NETWORK TO THE GRAPH EDGES. - # ----------------------------------------------------------------------------------------------- - for fc in ['road', 'rail', 'water', 'pipeline_crude_trf_rts', 'pipeline_prod_trf_rts', 'locations', 'intermodal', - 'locks']: - logger.debug("start: processing source and source_OID for: {}".format(fc)) - arcpy.DeleteField_management(os.path.join(scenario_gdb, fc), "source") - arcpy.DeleteField_management(os.path.join(scenario_gdb, fc), "source_OID") - arcpy.AddField_management(os.path.join(scenario_gdb, fc), "source", "TEXT") - arcpy.AddField_management(os.path.join(scenario_gdb, fc), "source_OID", "LONG") - arcpy.CalculateField_management(in_table=os.path.join(scenario_gdb, fc), field="source", - expression='"{}"'.format(fc), expression_type="PYTHON_9.3", code_block="") - arcpy.CalculateField_management(in_table=os.path.join(scenario_gdb, fc), field="source_OID", - expression="!OBJECTID!", expression_type="PYTHON_9.3", code_block="") - logger.debug("finish: processing source_OID for: {}".format(fc)) - - logger.debug("finish: hook_location_into_network") - - -# ============================================================================== - - -def cache_capacity_information(the_scenario, logger): - logger.info("start: cache_capacity_information") - - logger.debug( - "export the capacity, volume, and vcr data to the main.db for the locks, pipelines, and intermodal fcs") - - # need to cache out the pipeline, locks, and intermodal facility capacity information - # note source_oid is master_oid for pipeline capacities - # capacity_table: source, id_field_name, source_oid, capacity, volume, vcr - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - # drop the table - sql = "drop table if exists capacity_nodes" - main_db_con.execute(sql) - # create the table - sql = "create table capacity_nodes(" \ - "source text, " \ - "id_field_name text, " \ - "source_OID integer, " \ - "capacity real, " \ - "volume real, " \ - "vcr real" \ - ");" - main_db_con.execute(sql) - - for fc in ['locks', 'intermodal', 'pipeline_crude', 'pipeline_prod']: - capacity_update_list = [] # initialize the list at the beginning of every fc. - logger.debug("start: processing fc: {} ".format(fc)) - if 'pipeline' in fc: - id_field_name = 'MASTER_OID' - fields = ['MASTER_OID', 'Capacity', 'Volume', 'VCR'] - else: - id_field_name = 'source_OID' - fields = ['source_OID', 'Capacity', 'Volume', 'VCR'] - - # do the search cursor: - logger.debug("start: search cursor on: fc: {} with fields: {}".format(fc, fields)) - with arcpy.da.SearchCursor(os.path.join(the_scenario.main_gdb, fc), fields) as cursor: - for row in cursor: - source = fc - source_OID = row[0] - capacity = row[1] - volume = row[2] - vcr = row[3] - capacity_update_list.append([source, id_field_name, source_OID, capacity, volume, vcr]) - - logger.debug("start: execute many on: fc: {} with len: {}".format(fc, len(capacity_update_list))) - if len(capacity_update_list) > 0: - sql = "insert or ignore into capacity_nodes " \ - "(source, id_field_name, source_oid, capacity, volume, vcr) " \ - "values (?, ?, ?, ?, ?, ?);" - main_db_con.executemany(sql, capacity_update_list) - main_db_con.commit() - - # now get the mapping fields from the *_trf_rts and *_trf_sgmnts tables - # db table will have the form: source, id_field_name, id, mapping_id_field_name, mapping_id - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - # drop the table - sql = "drop table if exists pipeline_mapping" - main_db_con.execute(sql) - # create the table - sql = "create table pipeline_mapping(" \ - "source text, " \ - "id_field_name text, " \ - "id integer, " \ - "mapping_id_field_name text, " \ - "mapping_id integer);" - main_db_con.execute(sql) - - capacity_update_list = [] - for fc in ['pipeline_crude_trf_rts', 'pipeline_crude_trf_sgmts', 'pipeline_prod_trf_rts', - 'pipeline_prod_trf_sgmts']: - - logger.debug("start: processing fc: {} ".format(fc)) - if '_trf_rts' in fc: - id_field_name = 'source_OID' - mapping_id_field_name = 'tariff_ID' - fields = ['source_OID', 'tariff_ID'] - else: - id_field_name = 'tariff_ID' - mapping_id_field_name = 'MASTER_OID' - fields = ['tariff_ID', 'MASTER_OID'] - - # do the search cursor: - logger.debug("start: search cursor on: fc: {} with fields: {}".format(fc, fields)) - with arcpy.da.SearchCursor(os.path.join(the_scenario.main_gdb, fc), fields) as cursor: - for row in cursor: - - id = row[0] - mapping_id = row[1] - capacity_update_list.append([fc, id_field_name, id, mapping_id_field_name, mapping_id]) - - logger.debug("start: execute many on: fc: {} with len: {}".format(fc, len(capacity_update_list))) - - if len(capacity_update_list) > 0: - sql = "insert or ignore into pipeline_mapping " \ - "(source, " \ - "id_field_name, " \ - "id, " \ - "mapping_id_field_name, " \ - "mapping_id) " \ - "values (?, ?, ?, ?, ?);" - - main_db_con.executemany(sql, capacity_update_list) - main_db_con.commit() - - -# ============================================================================== - -def locations_add_links(logger, the_scenario, modal_layer_name, max_artificial_link_distance_miles): - - # ADD LINKS LOGIC - # first we near the mode to the locations fc - # then we iterate through the near table and build up a dictionary of links and all the near XYs on that link. - # then we split the links on the mode (except pipeline) and preserve the data of that link. - # then we near the locations to the nodes on the now split links. - # we ignore locations with near dist == 0 on those nodes. - # then we add the artificial link and note which locations got links. - # then we set the connects_to field if the location was connected. - - logger.debug("start: locations_add_links for mode: {}".format(modal_layer_name)) - - scenario_gdb = the_scenario.main_gdb - fp_to_modal_layer = os.path.join(scenario_gdb, "network", modal_layer_name) - - locations_fc = the_scenario.locations_fc - arcpy.DeleteField_management(fp_to_modal_layer, "LOCATION_ID") - arcpy.AddField_management(os.path.join(scenario_gdb, modal_layer_name), "LOCATION_ID", "long") - - arcpy.DeleteField_management(fp_to_modal_layer, "LOCATION_ID_NAME") - arcpy.AddField_management(os.path.join(scenario_gdb, modal_layer_name), "LOCATION_ID_NAME", "text") - - if float(max_artificial_link_distance_miles.strip(" Miles")) < 0.0000001: - logger.warning("Note: ignoring mode {}. User specified artificial link distance of {}".format( - modal_layer_name, max_artificial_link_distance_miles)) - logger.debug("Setting the definition query to artificial = 99999, so we get an empty dataset for the " - "make_feature_layer and subsequent near analysis") - - definition_query = "Artificial = 999999" # something to return an empty set - else: - definition_query = "Artificial = 0" # the normal def query. - - if "pipeline" in modal_layer_name: - - if arcpy.Exists(os.path.join(scenario_gdb, "network", fp_to_modal_layer + "_points")): - arcpy.Delete_management(os.path.join(scenario_gdb, "network", fp_to_modal_layer + "_points")) - - # limit near to end points - if arcpy.CheckProduct("ArcInfo") == "Available": - arcpy.FeatureVerticesToPoints_management(in_features=fp_to_modal_layer, - out_feature_class=fp_to_modal_layer + "_points", - point_location="BOTH_ENDS") - else: - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified feature vertices " - "process will be run.") - arcpy.AddGeometryAttributes_management(fp_to_modal_layer, "LINE_START_MID_END") - arcpy.MakeXYEventLayer_management(fp_to_modal_layer, "START_X", "START_Y", - "modal_start_points_lyr", LCC_PROJ) - - arcpy.MakeXYEventLayer_management(fp_to_modal_layer, "END_X", "END_Y", - "modal_end_points_lyr", LCC_PROJ) - - # Due to tool design, must define the feature class location and name slightly differently (separating - # scenario gdb from feature class name. fp_to_modal_layer is identical to scenario_gdb + "network" - # + modal_layer_name - arcpy.FeatureClassToFeatureClass_conversion("modal_start_points_lyr", - scenario_gdb, - os.path.join("network", modal_layer_name + "_points")) - - arcpy.Append_management(["modal_end_points_lyr"], - fp_to_modal_layer + "_points", "NO_TEST") - - arcpy.Delete_management("modal_start_points_lyr") - arcpy.Delete_management("modal_end_points_lyr") - arcpy.DeleteField_management(fp_to_modal_layer, "START_X") - arcpy.DeleteField_management(fp_to_modal_layer, "START_Y") - arcpy.DeleteField_management(fp_to_modal_layer, "MID_X") - arcpy.DeleteField_management(fp_to_modal_layer, "MID_Y") - arcpy.DeleteField_management(fp_to_modal_layer, "END_X") - arcpy.DeleteField_management(fp_to_modal_layer, "END_Y") - - logger.debug("start: make_feature_layer_management") - arcpy.MakeFeatureLayer_management(fp_to_modal_layer + "_points", "modal_lyr_" + modal_layer_name, - definition_query) - - else: - logger.debug("start: make_feature_layer_management") - arcpy.MakeFeatureLayer_management(fp_to_modal_layer, "modal_lyr_" + modal_layer_name, definition_query) - - logger.debug("adding links between locations_fc and mode {} with max dist of {}".format(modal_layer_name, - max_artificial_link_distance_miles)) - - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near")): - logger.debug("start: delete tmp near") - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near")) - - logger.debug("start: generate_near") - if arcpy.CheckProduct("ArcInfo") == "Available": - arcpy.GenerateNearTable_analysis(locations_fc, "modal_lyr_" + modal_layer_name, - os.path.join(scenario_gdb, "tmp_near"), - max_artificial_link_distance_miles, "LOCATION", "NO_ANGLE", "CLOSEST") - - else: - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified generate_near process " - "will be run.") - # Spatial Join - # Workaround for GenerateNearTable not being available for lower-level ArcGIS licenses. - - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_spatial_join")): - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join")) - - # First, add field to capture joined FID - arcpy.AddField_management("modal_lyr_" + modal_layer_name, "Join_FID", "LONG") - arcpy.CalculateField_management("modal_lyr_" + modal_layer_name, "Join_FID", "!OBJECTID!", - "PYTHON_9.3") - - arcpy.SpatialJoin_analysis(locations_fc, "modal_lyr_" + modal_layer_name, - os.path.join(scenario_gdb, "tmp_spatial_join"), - match_option="CLOSEST", search_radius=max_artificial_link_distance_miles) - - arcpy.DeleteField_management("modal_lyr_" + modal_layer_name, "Join_FID") - - # queryPointAndDistance on the original point and corresponding spatial join match - # For line in spatial_join: - result_dict = {} - - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_spatial_join"), - ["Target_FID", "Join_FID", "SHAPE@"]) as scursor1: - for row1 in scursor1: - with arcpy.da.SearchCursor("modal_lyr_" + modal_layer_name, - ["OBJECTID", "SHAPE@"]) as scursor2: - for row2 in scursor2: - if row1[1] == row2[0]: - if "pipeline" in modal_layer_name: - result = row2[1].angleAndDistanceTo(row1[2], "PLANAR") - # Capture the point geometry of the nearest point on the polyline to the location point - # and the minimum distance between the line and the point - # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist - result_dict[row1[0]] = [row1[1], row1[2], row2[1], result[1]] - else: - result = row2[1].queryPointAndDistance(row1[2], False) - # Capture the point geometry of the nearest point on the polyline to the location point - # and the minimum distance between the line and the point - # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist - result_dict[row1[0]] = [row1[1], row1[2], result[0], result[2]] - - # Write to a tmp_near table equivalent to what is create by Generate Near Table tool - arcpy.CreateTable_management(scenario_gdb, "tmp_near") - arcpy.AddField_management("tmp_near", "IN_FID", "LONG") - arcpy.AddField_management("tmp_near", "NEAR_FID", "LONG") - arcpy.AddField_management("tmp_near", "NEAR_DIST", "LONG") - arcpy.AddField_management("tmp_near", "FROM_X", "DOUBLE") - arcpy.AddField_management("tmp_near", "FROM_Y", "DOUBLE") - arcpy.AddField_management("tmp_near", "NEAR_X", "DOUBLE") - arcpy.AddField_management("tmp_near", "NEAR_Y", "DOUBLE") - - # insert the relevant data into the table - icursor = arcpy.da.InsertCursor("tmp_near", ['IN_FID', 'NEAR_FID', 'NEAR_DIST', 'FROM_X', 'FROM_Y', - 'NEAR_X', 'NEAR_Y']) - - for in_fid in result_dict: - near_fid = result_dict[in_fid][0] - near_distance = result_dict[in_fid][3] - from_x = result_dict[in_fid][1].firstPoint.X - from_y = result_dict[in_fid][1].firstPoint.Y - near_x = result_dict[in_fid][2].firstPoint.X - near_y = result_dict[in_fid][2].firstPoint.Y - icursor.insertRow([in_fid, near_fid, near_distance, from_x, from_y, near_x, near_y]) - - del icursor - - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join")) - - edit = arcpy.da.Editor(os.path.join(scenario_gdb)) - edit.startEditing(False, False) - edit.startOperation() - - id_fieldname = arcpy.Describe(os.path.join(scenario_gdb, modal_layer_name)).OIDFieldName - - seenids = {} - - # SPLIT LINKS LOGIC - # 1) first search through the tmp_near fc and add points from the near on that link. - # 2) next we query the mode layer and get the mode specific data using the near FID. - # 3) then we split the old link, and use insert cursor to populate mode specific data into fc for the two new links. - # 4) then we delete the old unsplit link - logger.debug("start: split links") - - if arcpy.CheckProduct("ArcInfo") != "Available": - # Adding warning here rather than within the search cursor loop - logger.warning( - "The Advanced/ArcInfo license level of ArcGIS is not available. Modified split links process " - "will be run.") - - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_near"), - ["NEAR_FID", "NEAR_X", "NEAR_Y", "NEAR_DIST"]) as scursor: - - for row in scursor: - - # if the near distance is 0, then its connected and we don't need to split the line - if row[3] == 0: - # only give debug warning if not pipeline - if "pipeline" not in modal_layer_name: - logger.warning( - "Split links code: LOCATION MIGHT BE ON THE NETWORK. Ignoring NEAR_FID {} with NEAR_DIST {}".format( - row[0], row[3])) - - if not row[3] == 0: - - # STEP 1: point geoms where to split from the near XY - # --------------------------------------------------- - # get the line ID to split - theIdToGet = str(row[0]) # this is the link id we need - - if not theIdToGet in seenids: - seenids[theIdToGet] = [] - - point = arcpy.Point() - point.X = float(row[1]) - point.Y = float(row[2]) - point_geom = arcpy.PointGeometry(point, ftot_supporting_gis.LCC_PROJ) - seenids[theIdToGet].append(point_geom) - - # STEP 2 -- get mode specific data from the link - # ------------------------------------------------ - if 'pipeline' not in modal_layer_name: - - for theIdToGet in seenids: - - # initialize the variables so we dont get any gremlins - in_line = None # the shape geometry - in_capacity = None # road + rail - in_volume = None # road + rail - in_vcr = None # road + rail | volume to capacity ratio - in_fclass = None # road | fclass - in_speed = None # road | rounded speed - in_stracnet = None # rail - in_density_code = None # rail - in_tot_up_dwn = None # water - - if modal_layer_name == 'road': - for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), - ["SHAPE@", "Capacity", "Volume", "VCR", "FCLASS", "ROUNDED_SPEED"], - where_clause=id_fieldname + " = " + theIdToGet): - in_line = row[0] - in_capacity = row[1] - in_volume = row[2] - in_vcr = row[3] - in_fclass = row[4] - in_speed = row[5] - - if modal_layer_name == 'rail': - for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), - ["SHAPE@", "Capacity", "Volume", "VCR", "STRACNET", - "DENSITY_CODE"], where_clause=id_fieldname + " = " + theIdToGet): - in_line = row[0] - in_capacity = row[1] - in_volume = row[2] - in_vcr = row[3] - in_stracnet = row[4] - in_density_code = row[5] - - if modal_layer_name == 'water': - for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), - ["SHAPE@", "Capacity", "Volume", "VCR", "TOT_UP_DWN"], - where_clause=id_fieldname + " = " + theIdToGet): - in_line = row[0] - in_capacity = row[1] - in_volume = row[2] - in_vcr = row[3] - in_tot_up_dwn = row[4] - - # STEP 3: Split and populate with mode specific data from old link - # ------------------------------------------------------------------ - if arcpy.CheckProduct("ArcInfo") == "Available": - split_lines = arcpy.management.SplitLineAtPoint(in_line, seenids[theIdToGet], arcpy.Geometry(), 1) - - else: - # This is the alternative approach for those without an Advanced/ArcInfo license - point_list = seenids[theIdToGet] - line_list = [in_line] - split_lines = [] - continue_iteration = 'continue running' - - while continue_iteration == 'continue running': - line_list, point_list, split_lines, continue_iteration = cut_lines(line_list, point_list, split_lines) - - if not len(split_lines) == 1: - - # ROAD - if modal_layer_name == 'road': - - icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), - ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'FCLASS', - 'ROUNDED_SPEED', 'Volume', 'Capacity', 'VCR']) - - # Insert new links that include the mode-specific attributes - for new_line in split_lines: - len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude - icursor.insertRow( - [new_line, 0, modal_layer_name, len_in_miles, in_fclass, in_speed, in_volume, - in_capacity, in_vcr]) - - # Delete cursor object - del icursor - - elif modal_layer_name == 'rail': - icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), - ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'STRACNET', - 'DENSITY_CODE', 'Volume', 'Capacity', 'VCR']) - - # Insert new rows that include the mode-specific attributes - for new_line in split_lines: - len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude - icursor.insertRow( - [new_line, 0, modal_layer_name, len_in_miles, in_stracnet, in_density_code, in_volume, - in_capacity, in_vcr]) - - # Delete cursor object - del icursor - - elif modal_layer_name == 'water': - - icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), - ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'TOT_UP_DWN', - 'Volume', 'Capacity', 'VCR']) - - # Insert new rows that include the mode-specific attributes - for new_line in split_lines: - len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude - icursor.insertRow( - [new_line, 0, modal_layer_name, len_in_miles, in_tot_up_dwn, in_volume, in_capacity, - in_vcr]) - - # Delete cursor object - del icursor - - else: - logger.warning("Modal_layer_name: {} is not supported.".format(modal_layer_name)) - - # STEP 4: Delete old unsplit data - with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, modal_layer_name), ['OID@'], - where_clause=id_fieldname + " = " + theIdToGet) as ucursor: - for row in ucursor: - ucursor.deleteRow() - - # if the split doesn't work - else: - logger.detailed_debug( - "the line split didn't work for ID: {}. " - "Might want to investigate. " - "Could just be an artifact from the near result being the end of a line.".format( - theIdToGet)) - - edit.stopOperation() - edit.stopEditing(True) - - # delete the old features - # ------------------------ - logger.debug("start: delete old features (tmp_near, tmp_near_2, tmp_nodes)") - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near")): - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near")) - - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near_2")): - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near_2")) - - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_nodes")): - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_nodes")) - - # add artificial links - # now that the lines have been split add lines from the from points to the nearest node - # -------------------------------------------------------------------------------------- - logger.debug("start: add artificial links now w/ definition_query: {}".format(definition_query)) - logger.debug("start: make_featurelayer 2") - fp_to_modal_layer = os.path.join(scenario_gdb, "network", modal_layer_name) - arcpy.MakeFeatureLayer_management(fp_to_modal_layer, "modal_lyr_" + modal_layer_name + "2", definition_query) - logger.debug("start: feature vertices to points 2") - if arcpy.CheckProduct("ArcInfo") == "Available": - arcpy.FeatureVerticesToPoints_management(in_features="modal_lyr_" + modal_layer_name + "2", - out_feature_class=os.path.join(scenario_gdb, "tmp_nodes"), - point_location="BOTH_ENDS") - else: - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified feature vertices " - "process will be run.") - arcpy.AddGeometryAttributes_management("modal_lyr_" + modal_layer_name + "2", "LINE_START_MID_END") - arcpy.MakeXYEventLayer_management("modal_lyr_" + modal_layer_name + "2", "START_X", "START_Y", - "modal_start_points_lyr", LCC_PROJ) - - arcpy.MakeXYEventLayer_management("modal_lyr_" + modal_layer_name + "2", "END_X", "END_Y", - "modal_end_points_lyr", LCC_PROJ) - - arcpy.FeatureClassToFeatureClass_conversion("modal_start_points_lyr", - scenario_gdb, - "tmp_nodes") - - arcpy.Append_management(["modal_end_points_lyr"], - "tmp_nodes", "NO_TEST") - - arcpy.Delete_management("modal_start_points_lyr") - arcpy.Delete_management("modal_end_points_lyr") - arcpy.DeleteField_management(fp_to_modal_layer, "START_X") - arcpy.DeleteField_management(fp_to_modal_layer, "START_Y") - arcpy.DeleteField_management(fp_to_modal_layer, "MID_X") - arcpy.DeleteField_management(fp_to_modal_layer, "MID_Y") - arcpy.DeleteField_management(fp_to_modal_layer, "END_X") - arcpy.DeleteField_management(fp_to_modal_layer, "END_Y") - - logger.debug("start: generate near table 2") - if arcpy.CheckProduct("ArcInfo") == "Available": - arcpy.GenerateNearTable_analysis(locations_fc, os.path.join(scenario_gdb, "tmp_nodes"), - os.path.join(scenario_gdb, "tmp_near_2"), - max_artificial_link_distance_miles, "LOCATION", "NO_ANGLE", "CLOSEST") - - else: - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified generate_near process " - "will be run.") - # Spatial Join - # Workaround for GenerateNearTable not being available for lower-level ArcGIS licenses - - if arcpy.Exists(os.path.join(scenario_gdb, "tmp_spatial_join_2")): - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join_2")) - - # First, add field to capture joined FID - arcpy.AddField_management("tmp_nodes", "Join_FID", "LONG") - arcpy.CalculateField_management("tmp_nodes", "Join_FID", "!OBJECTID!", "PYTHON_9.3") - - arcpy.SpatialJoin_analysis(locations_fc, "tmp_nodes", - os.path.join(scenario_gdb, "tmp_spatial_join_2"), - match_option="CLOSEST", search_radius=max_artificial_link_distance_miles) - - # queryPointAndDistance on the original point and corresponding spatial join match - # For line in spatial_join: - result_dict = {} - - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_spatial_join_2"), - ["Target_FID", "Join_FID", "SHAPE@"]) as scursor1: - for row1 in scursor1: - with arcpy.da.SearchCursor("tmp_nodes", - ["OBJECTID", "SHAPE@"]) as scursor2: - for row2 in scursor2: - if row1[1] == row2[0]: - result = row2[1].angleAndDistanceTo(row1[2], "PLANAR") - # Capture the point geometry of the nearest point on the polyline to the location point - # and the minimum distance between the line and the point - # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist - result_dict[row1[0]] = [row1[1], row1[2], row2[1], result[1]] - - # Write to a tmp_near table equivalent to what is create by Generate Near Table tool - arcpy.CreateTable_management(scenario_gdb, "tmp_near_2") - arcpy.AddField_management("tmp_near_2", "IN_FID", "LONG") - arcpy.AddField_management("tmp_near_2", "NEAR_FID", "LONG") - arcpy.AddField_management("tmp_near_2", "NEAR_DIST", "LONG") - arcpy.AddField_management("tmp_near_2", "FROM_X", "DOUBLE") - arcpy.AddField_management("tmp_near_2", "FROM_Y", "DOUBLE") - arcpy.AddField_management("tmp_near_2", "NEAR_X", "DOUBLE") - arcpy.AddField_management("tmp_near_2", "NEAR_Y", "DOUBLE") - - # insert the relevant data into the table - icursor = arcpy.da.InsertCursor("tmp_near_2", ['IN_FID', 'NEAR_FID', 'NEAR_DIST', 'FROM_X', 'FROM_Y', - 'NEAR_X', 'NEAR_Y']) - - for in_fid in result_dict: - near_fid = result_dict[in_fid][0] - near_distance = result_dict[in_fid][3] - from_x = result_dict[in_fid][1].firstPoint.X - from_y = result_dict[in_fid][1].firstPoint.Y - near_x = result_dict[in_fid][2].firstPoint.X - near_y = result_dict[in_fid][2].firstPoint.Y - icursor.insertRow([in_fid, near_fid, near_distance, from_x, from_y, near_x, near_y]) - - del icursor - - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join_2")) - - logger.debug("start: delete tmp_nodes") - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_nodes")) - - logger.debug("start: start editor") - edit = arcpy.da.Editor(os.path.join(scenario_gdb)) - edit.startEditing(False, False) - edit.startOperation() - - icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), - ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'LOCATION_ID', - 'LOCATION_ID_NAME']) # add location_id for setting flow restrictions - - location_id_name_dict = get_location_id_name_dict(the_scenario, logger) - connected_location_ids = [] - connected_location_id_names = [] - logger.debug("start: search cursor on tmp_near_2") - with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_near_2"), - ["FROM_X", "FROM_Y", "NEAR_X", "NEAR_Y", "NEAR_DIST", "IN_FID"]) as scursor: - - for row in scursor: - - if not row[4] == 0: - - # use the unique objectid (in_fid) from the near to determine - # if we have an in or an out location. - # then set the flow restrictions appropriately. - - in_fid = row[5] - location_id_name = location_id_name_dict[in_fid] - location_id = location_id_name.split("_")[0] - connected_location_ids.append(location_id) - connected_location_id_names.append(location_id_name) - - coordList = [] - coordList.append(arcpy.Point(row[0], row[1])) - coordList.append(arcpy.Point(row[2], row[3])) - polyline = arcpy.Polyline(arcpy.Array(coordList)) - - len_in_miles = Q_(polyline.length, "meters").to("miles").magnitude - - # insert artificial link attributes - icursor.insertRow([polyline, 1, modal_layer_name, len_in_miles, location_id, location_id_name]) - - else: - logger.warning("Artificial Link code: Ignoring NEAR_FID {} with NEAR_DIST {}".format(row[0], row[4])) - - del icursor - logger.debug("start: stop editing") - edit.stopOperation() - edit.stopEditing(True) - - arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near_2")) - - # ALSO SET CONNECTS_X FIELD IN POINT LAYER - # ----------------------------------------- - logger.debug("start: connect_x") - - edit = arcpy.da.Editor(scenario_gdb) - edit.startEditing(False, False) - edit.startOperation() - with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, locations_fc), - ["LOCATION_ID_NAME", "connects_" + modal_layer_name]) as cursor: - - for row in cursor: - - if row[0] in connected_location_id_names: - row[1] = 1 - cursor.updateRow(row) - - edit.stopOperation() - edit.stopEditing(True) - - logger.debug("finish: locations_add_links") - - -# ============================================================================== - - -def ignore_locations_not_connected_to_network(the_scenario, logger): - logger.info("start: ignore_locations_not_connected_to_network") - logger.debug("flag locations which don't connect to the network") - - # flag locations which dont connect to the network in the GIS - # ----------------------------------------------------------- - - # add the ignore field to the fc - scenario_gdb = the_scenario.main_gdb - locations_fc = the_scenario.locations_fc - - edit = arcpy.da.Editor(scenario_gdb) - edit.startEditing(False, False) - edit.startOperation() - - query = "connects_road = 0 and " \ - "connects_rail = 0 and " \ - "connects_water = 0 and " \ - "connects_pipeline_prod_trf_rts = 0 and " \ - "connects_pipeline_crude_trf_rts = 0" - - list_of_ignored_locations = [] - with arcpy.da.UpdateCursor(locations_fc, ['location_id', 'ignore'], where_clause=query) as ucursor: - for row in ucursor: - list_of_ignored_locations.append(row[0]) - logger.debug('Location ID {} does not connect to the network and will be ignored'.format(row[0])) - row[1] = 1 - ucursor.updateRow(row) - - if len(list_of_ignored_locations) > 0: - - logger.result("# of locations not connected to network and ignored: \t{}".format(len( - list_of_ignored_locations)/2.0)) - logger.info("note: check the log files for additional debug information.") - - edit.stopOperation() - edit.stopEditing(True) - - # flag locations which dont connect to the network in the DB - # ----------------------------------------------------------- - scenario_db = the_scenario.main_db - - if os.path.exists(scenario_db): - - with sqlite3.connect(scenario_db) as db_con: - - logger.debug("connected to the db") - - db_cur = db_con.cursor() - - # iterate through the locations table and set ignore flag - # if location_id is in the list_of_ignored_facilities - # ------------------------------------------------------------ - - logger.debug("setting ignore fields for location in the DB locations and facilities tables") - - sql = "select location_id, ignore_location from locations;" - - db_cur.execute(sql) - for row in db_cur: - - if str(row[0]) in list_of_ignored_locations: # cast as string since location_id field is a string - ignore_flag = 'network' - else: - ignore_flag = 'false' - sql = "update locations set ignore_location = '{}' where location_id = '{}';".format(ignore_flag, - row[0]) - db_con.execute(sql) - - # iterate through the facilities table, - # and check if location_id is in the list_of_ignored_facilities - # ------------------------------------------------------------ - sql = "update facilities set ignore_facility = '{}' where location_id = '{}';".format(ignore_flag, - row[0]) - - db_con.execute(sql) - - logger.debug("finished: ignore_locations_not_connected_to_network") - return 1 - -# ====================================================================================================================== - -# this code subsets the road network so that only links within the minimum bounding geometry (MBG) buffer are -# preserved. This method ends with the arcpy.Compact_management() call that reduces the size of the geodatabase and -# increases overall performance when hooking facilities into the network, and exporting the road fc to a shapefile -# for networkX. - - -def minimum_bounding_geometry(the_scenario, logger): - logger.info("start: minimum_bounding_geometry") - arcpy.env.workspace = the_scenario.main_gdb - - # Clean up any left of layers from a previous run - if arcpy.Exists("road_lyr"): - arcpy.Delete_management("road_lyr") - if arcpy.Exists("rail_lyr"): - arcpy.Delete_management("rail_lyr") - if arcpy.Exists("water_lyr"): - arcpy.Delete_management("water_lyr") - if arcpy.Exists("pipeline_prod_trf_rts_lyr"): - arcpy.Delete_management("pipeline_prod_trf_rts_lyr") - if arcpy.Exists("pipeline_crude_trf_rts_lyr"): - arcpy.Delete_management("pipeline_crude_trf_rts_lyr") - if arcpy.Exists("Locations_MBG"): - arcpy.Delete_management("Locations_MBG") - if arcpy.Exists("Locations_MBG_Buffered"): - arcpy.Delete_management("Locations_MBG_Buffered") - - # Determine the minimum bounding geometry of the scenario - # The advanced license is required to use the convex hull method. If not available, default to rectangle_by_area - # which will not subset things quite as small but is still better than no subsetting at all - if arcpy.CheckProduct("ArcInfo") == "Available": - arcpy.MinimumBoundingGeometry_management("Locations", "Locations_MBG", "CONVEX_HULL") - - else: - arcpy.MinimumBoundingGeometry_management("Locations", "Locations_MBG", "RECTANGLE_BY_AREA") - logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. A slight modification to " - "the minimum bounding geometry process is necessary to ensure FTOT can successfully run.") - - # Buffer the minimum bounding geometry of the scenario - arcpy.Buffer_analysis("Locations_MBG", "Locations_MBG_Buffered", "100 Miles", "FULL", "ROUND", "NONE", "", - "GEODESIC") - - # Select the roads within the buffer - # ----------------------------------- - arcpy.MakeFeatureLayer_management("road", "road_lyr") - arcpy.SelectLayerByLocation_management("road_lyr", "INTERSECT", "Locations_MBG_Buffered") - - result = arcpy.GetCount_management("road") - count_all_roads = float(result.getOutput(0)) - - result = arcpy.GetCount_management("road_lyr") - count_roads_subset = float(result.getOutput(0)) - - ## CHANGE - if count_all_roads > 0: - roads_percentage = count_roads_subset / count_all_roads - else: - roads_percentage = 0 - - # Only subset if the subset will result in substantial reduction of the road network size - if roads_percentage < 0.75: - # Switch selection to identify what's outside the buffer - arcpy.SelectLayerByAttribute_management("road_lyr", "SWITCH_SELECTION") - - # Add in FC 1 roadways (going to keep all interstate highways) - arcpy.SelectLayerByAttribute_management("road_lyr", "REMOVE_FROM_SELECTION", "FCLASS = 1") - - # Delete the features outside the buffer - with arcpy.da.UpdateCursor('road_lyr', ['OBJECTID']) as ucursor: - for ucursor_row in ucursor: - ucursor.deleteRow() - - arcpy.Delete_management("road_lyr") - - # # Select the rail within the buffer - # # --------------------------------- - - arcpy.Delete_management("Locations_MBG") - arcpy.Delete_management("Locations_MBG_Buffered") - - # finally, compact the geodatabase so the MBG has an effect on runtime. - arcpy.Compact_management(the_scenario.main_gdb) - logger.debug("finish: minimum_bounding_geometry") - +# ------------------------------------------------------------------------------ +# ftot_routing.py +# Purpose: the purpose of this module is to clean up, +# create the locations_fc, +# hook locations into the network, +# ignore locations not connected to the network, +# export capacity information to the main.db, +# export the assets from GIS export_fcs_from_main_gdb + +# Revised: 1/15/19 - MNP +# ------------------------------------------------------------------------------ + +import os +import arcpy +import sqlite3 +import ftot_supporting_gis +from ftot import Q_ + +LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') + + +# ======================================================================== + + +def connectivity(the_scenario, logger): + + checks_and_cleanup(the_scenario, logger) + + # create the locations_fc + create_locations_fc(the_scenario, logger) + + # use MBG to subset the road network to a buffer around the locations FC + minimum_bounding_geometry(the_scenario, logger) + + # hook locations into the network + hook_locations_into_network(the_scenario, logger) + + # ignore locations not connected to the network + ignore_locations_not_connected_to_network(the_scenario, logger) + + # report out material missing after connecting to the network + from ftot_facilities import db_report_commodity_potentials + db_report_commodity_potentials(the_scenario, logger) + + # export capacity information to the main.db + cache_capacity_information(the_scenario, logger) + + +# ========================================================================= + + +def checks_and_cleanup(the_scenario, logger): + logger.info("start: checks_and_cleanup") + + scenario_gdb = the_scenario.main_gdb + if not os.path.exists(scenario_gdb): + error = "can't find scenario gdb {}".format(scenario_gdb) + raise IOError(error) + + # check for scenario DB + # --------------------------------- + scenario_db = the_scenario.main_db + if not arcpy.Exists(scenario_db): + raise Exception("scenario_db not found {} ".format(scenario_db)) + + logger.debug("finish: checks_and_cleanup") + + +# ============================================================================== + + +def create_locations_fc(the_scenario, logger): + logger.info("start: create_locations_fc") + co_location_offet = 0.1 + logger.debug("co-location offset is necessary to prevent the locations from being treated as intermodal " + "facilities.") + logger.debug("collocation off-set: {} meters".format(co_location_offet)) + + locations_fc = the_scenario.locations_fc + + # delete the old location FC before we create it + from ftot_facilities import gis_clear_feature_class + gis_clear_feature_class(locations_fc, logger) + + # create the feature class + arcpy.CreateFeatureclass_management(the_scenario.main_gdb, "locations", "POINT", "#", "DISABLED", "DISABLED", + ftot_supporting_gis.LCC_PROJ) + + # add the location_id field + arcpy.AddField_management(locations_fc, "location_id", "TEXT") + + # add the location_id_name field + arcpy.AddField_management(locations_fc, "location_id_name", "TEXT") + + # add the connects_road field + arcpy.AddField_management(locations_fc, "connects_road", "SHORT") + + # add the connects_rail field + arcpy.AddField_management(locations_fc, "connects_rail", "SHORT") + + # add the connects_water field + arcpy.AddField_management(locations_fc, "connects_water", "SHORT") + + # add the connects_pipeline_prod field + arcpy.AddField_management(locations_fc, "connects_pipeline_prod_trf_rts", "SHORT") + + # add the connects_pipeline_crude field + arcpy.AddField_management(locations_fc, "connects_pipeline_crude_trf_rts", "SHORT") + + # add the ignore field + arcpy.AddField_management(locations_fc, "ignore", "SHORT") + + # start an edit session for the insert cursor + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + # create insert cursor + with arcpy.da.InsertCursor(locations_fc, ["location_id_name", "location_id", "SHAPE@"]) as insert_cursor: + + # loop through DB and populate the fc + with sqlite3.connect(the_scenario.main_db) as db_con: + + sql = "select * from locations;" + db_cur = db_con.execute(sql) + for row in db_cur: + location_id = row[0] + + # create a point for each location "out" + location_point = arcpy.Point() + location_point.X = row[1] + co_location_offet + location_point.Y = row[2] + co_location_offet + location_point_geom = arcpy.PointGeometry(location_point, LCC_PROJ) + + insert_cursor.insertRow([str(location_id) + "_OUT", location_id, location_point_geom]) + + # create a point for each location "in" + location_point = arcpy.Point() + location_point.X = row[1] - co_location_offet + location_point.Y = row[2] - co_location_offet + location_point_geom = arcpy.PointGeometry(location_point, LCC_PROJ) + + insert_cursor.insertRow([str(location_id) + "_IN", location_id, location_point_geom]) + + edit.stopOperation() + edit.stopEditing(True) + + logger.debug("finish: create_locations_fc") + + +# ============================================================================== + + +def get_xy_location_id_dict(the_scenario, logger): + logger.debug("start: get_xy_location_id_dict") + + with sqlite3.connect(the_scenario.main_db) as db_con: + + sql = "select location_id, shape_x, shape_y from locations;" + db_cur = db_con.execute(sql) + + xy_location_id_dict = {} + for row in db_cur: + location_id = row[0] + shape_x = row[1] + shape_y = row[2] + xy_location_id_dict[location_id] = "[{}, {}]".format(shape_x, shape_y) + + logger.debug("finish: get_xy_location_id_dict") + + return xy_location_id_dict + + +# ============================================================================== + + +def get_location_id_name_dict(the_scenario, logger): + logger.debug("start: get_location_id_name_dict") + + location_id_name_dict = {} + + with arcpy.da.SearchCursor(the_scenario.locations_fc, ["location_id_name", "OBJECTID"]) as scursor: + + for row in scursor: + location_id_name = row[0] + objectid = row[1] + # shape = row[2] + location_id_name_dict[objectid] = location_id_name + + logger.debug("finish: get_location_id_name_dict") + + return location_id_name_dict + + +# =============================================================================== + + +def delete_old_artificial_link(the_scenario, logger): + logger.debug("start: delete_old_artificial_link") + for mode in ["road", "rail", "water", "pipeline_crude_trf_rts", "pipeline_prod_trf_rts"]: + edit = arcpy.da.Editor(the_scenario.main_gdb) + edit.startEditing(False, False) + edit.startOperation() + + with arcpy.da.UpdateCursor(os.path.join(the_scenario.main_gdb, 'network', mode), ["artificial"], + where_clause="Artificial = 1") as cursor: + for row in cursor: + cursor.deleteRow() + + edit.stopOperation() + edit.stopEditing(True) + logger.debug("finish: delete_old_artificial_link") + + +# =============================================================================== + + +def cut_lines(line_list, point_list, split_lines): + for line in line_list: + is_cut = "Not Cut" + if line.length > 0.0: # Make sure it's not an empty geometry. + for point in point_list: + # Even "coincident" points can show up as spatially non-coincident in their + # floating-point XY values, so we set up a tolerance. + if line.distanceTo(point) < 1.0: + # To ensure coincidence, snap the point to the line before proceeding. + snap_point = line.snapToLine(point).firstPoint + # Make sure the point isn't on a line endpoint, otherwise cutting will produce + # an empty geometry. + if not (snap_point.equals(line.lastPoint) and snap_point.equals(line.firstPoint)): + # Cut the line. Try it a few different ways to try increase the likelihood it will actually cut + cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( + [arcpy.Point(snap_point.X + 10.0, snap_point.Y + 10.0), + arcpy.Point(snap_point.X - 10.0, snap_point.Y - 10.0)]), LCC_PROJ)) + if cut_line_1.length == 0 or cut_line_2.length == 0: + cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( + [arcpy.Point(snap_point.X - 10.0, snap_point.Y + 10.0), + arcpy.Point(snap_point.X + 10.0, snap_point.Y - 10.0)]), LCC_PROJ)) + if cut_line_1.length == 0 or cut_line_2.length == 0: + cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( + [arcpy.Point(snap_point.X + 10.0, snap_point.Y), + arcpy.Point(snap_point.X - 10.0, snap_point.Y)]), LCC_PROJ)) + if cut_line_1.length == 0 or cut_line_2.length == 0: + cut_line_1, cut_line_2 = line.cut(arcpy.Polyline(arcpy.Array( + [arcpy.Point(snap_point.X, snap_point.Y + 10.0), + arcpy.Point(snap_point.X, snap_point.Y - 10.0)]), LCC_PROJ)) + # Make sure both descendents have non-zero geometry. + if cut_line_1.length > 0.0 and cut_line_2.length > 0.0: + # Feed the cut lines back into the "line" list as candidates to be cut again. + line_list.append(cut_line_1) + line_list.append(cut_line_2) + line_list.remove(line) + # The cut loop will only exit when all lines cannot be cut smaller without producing + # zero-length geometries + is_cut = "Cut" + # break the loop because we've cut a line into two now and need to start over. + break + point_list.remove(point) + + if is_cut == "Not Cut" and len(point_list) == 0: + split_lines.append(line) + line_list.remove(line) + + if len(line_list) == 0 and len(point_list) == 0: + continue_iteration = 'done' + else: + continue_iteration = 'continue running' + return line_list, point_list, split_lines, continue_iteration + +# =============================================================================== + + +def hook_locations_into_network(the_scenario, logger): + + # Add artificial links from the locations feature class into the network + # ----------------------------------------------------------------------- + logger.info("start: hook_location_into_network") + + scenario_gdb = the_scenario.main_gdb + if not os.path.exists(scenario_gdb): + error = "can't find scenario gdb {}".format(scenario_gdb) + raise IOError(error) + + # LINKS TO/FROM LOCATIONS + # --------------------------- + road_max_artificial_link_distance_miles = str(the_scenario.road_max_artificial_link_dist.magnitude) + " Miles" + rail_max_artificial_link_distance_miles = str(the_scenario.rail_max_artificial_link_dist.magnitude) + " Miles" + water_max_artificial_link_distance_miles = str(the_scenario.water_max_artificial_link_dist.magnitude) + " Miles" + pipeline_crude_max_artificial_link_distance_miles = str(the_scenario.pipeline_crude_max_artificial_link_dist.magnitude) + " Miles" + pipeline_prod_max_artificial_link_distance_miles = str(the_scenario.pipeline_prod_max_artificial_link_dist.magnitude) + " Miles" + + # cleanup any old artificial links + delete_old_artificial_link(the_scenario, logger) + + # LINKS TO/FROM LOCATIONS + # --------------------------- + locations_fc = the_scenario.locations_fc + + arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_road", "SHORT") + arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_road", 0, "PYTHON_9.3") + + arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_rail", "SHORT") + arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_rail", 0, "PYTHON_9.3") + + arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_water", "SHORT") + arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_water", 0, "PYTHON_9.3") + + arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_prod_trf_rts", "SHORT") + arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_prod_trf_rts", 0, + "PYTHON_9.3") + + arcpy.AddField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_crude_trf_rts", "SHORT") + arcpy.CalculateField_management(os.path.join(scenario_gdb, locations_fc), "connects_pipeline_crude_trf_rts", 0, + "PYTHON_9.3") + + # check for permitted modes before creating artificial links + from ftot_networkx import check_permitted_modes + check_permitted_modes(the_scenario, logger) + + if 'road' in the_scenario.permittedModes: + locations_add_links(logger, the_scenario, "road", road_max_artificial_link_distance_miles) + if 'rail' in the_scenario.permittedModes: + locations_add_links(logger, the_scenario, "rail", rail_max_artificial_link_distance_miles) + if 'water' in the_scenario.permittedModes: + locations_add_links(logger, the_scenario, "water", water_max_artificial_link_distance_miles) + if 'pipeline_crude_trf_rts' in the_scenario.permittedModes: + locations_add_links(logger, the_scenario, "pipeline_crude_trf_rts", + pipeline_crude_max_artificial_link_distance_miles) + if 'pipeline_prod_trf_rts' in the_scenario.permittedModes: + locations_add_links(logger, the_scenario, "pipeline_prod_trf_rts", + pipeline_prod_max_artificial_link_distance_miles) + + # ADD THE SOURCE AND SOURCE_OID FIELDS SO WE CAN MAP THE LINKS IN THE NETWORK TO THE GRAPH EDGES. + # ----------------------------------------------------------------------------------------------- + for fc in ['road', 'rail', 'water', 'pipeline_crude_trf_rts', 'pipeline_prod_trf_rts', 'locations', 'intermodal', + 'locks']: + logger.debug("start: processing source and source_OID for: {}".format(fc)) + arcpy.DeleteField_management(os.path.join(scenario_gdb, fc), "source") + arcpy.DeleteField_management(os.path.join(scenario_gdb, fc), "source_OID") + arcpy.AddField_management(os.path.join(scenario_gdb, fc), "source", "TEXT") + arcpy.AddField_management(os.path.join(scenario_gdb, fc), "source_OID", "LONG") + arcpy.CalculateField_management(in_table=os.path.join(scenario_gdb, fc), field="source", + expression='"{}"'.format(fc), expression_type="PYTHON_9.3", code_block="") + arcpy.CalculateField_management(in_table=os.path.join(scenario_gdb, fc), field="source_OID", + expression="!OBJECTID!", expression_type="PYTHON_9.3", code_block="") + logger.debug("finish: processing source_OID for: {}".format(fc)) + + logger.debug("finish: hook_location_into_network") + + +# ============================================================================== + + +def cache_capacity_information(the_scenario, logger): + logger.info("start: cache_capacity_information") + + logger.debug( + "export the capacity, volume, and vcr data to the main.db for the locks, pipelines, and intermodal fcs") + + # need to cache out the pipeline, locks, and intermodal facility capacity information + # note source_oid is master_oid for pipeline capacities + # capacity_table: source, id_field_name, source_oid, capacity, volume, vcr + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + # drop the table + sql = "drop table if exists capacity_nodes" + main_db_con.execute(sql) + # create the table + sql = "create table capacity_nodes(" \ + "source text, " \ + "id_field_name text, " \ + "source_OID integer, " \ + "capacity real, " \ + "volume real, " \ + "vcr real" \ + ");" + main_db_con.execute(sql) + + for fc in ['locks', 'intermodal', 'pipeline_crude', 'pipeline_prod']: + capacity_update_list = [] # initialize the list at the beginning of every fc. + logger.debug("start: processing fc: {} ".format(fc)) + if 'pipeline' in fc: + id_field_name = 'MASTER_OID' + fields = ['MASTER_OID', 'Capacity', 'Volume', 'VCR'] + else: + id_field_name = 'source_OID' + fields = ['source_OID', 'Capacity', 'Volume', 'VCR'] + + # do the search cursor: + logger.debug("start: search cursor on: fc: {} with fields: {}".format(fc, fields)) + with arcpy.da.SearchCursor(os.path.join(the_scenario.main_gdb, fc), fields) as cursor: + for row in cursor: + source = fc + source_OID = row[0] + capacity = row[1] + volume = row[2] + vcr = row[3] + capacity_update_list.append([source, id_field_name, source_OID, capacity, volume, vcr]) + + logger.debug("start: execute many on: fc: {} with len: {}".format(fc, len(capacity_update_list))) + if len(capacity_update_list) > 0: + sql = "insert or ignore into capacity_nodes " \ + "(source, id_field_name, source_oid, capacity, volume, vcr) " \ + "values (?, ?, ?, ?, ?, ?);" + main_db_con.executemany(sql, capacity_update_list) + main_db_con.commit() + + # now get the mapping fields from the *_trf_rts and *_trf_sgmnts tables + # db table will have the form: source, id_field_name, id, mapping_id_field_name, mapping_id + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + # drop the table + sql = "drop table if exists pipeline_mapping" + main_db_con.execute(sql) + # create the table + sql = "create table pipeline_mapping(" \ + "source text, " \ + "id_field_name text, " \ + "id integer, " \ + "mapping_id_field_name text, " \ + "mapping_id integer);" + main_db_con.execute(sql) + + capacity_update_list = [] + for fc in ['pipeline_crude_trf_rts', 'pipeline_crude_trf_sgmts', 'pipeline_prod_trf_rts', + 'pipeline_prod_trf_sgmts']: + + logger.debug("start: processing fc: {} ".format(fc)) + if '_trf_rts' in fc: + id_field_name = 'source_OID' + mapping_id_field_name = 'tariff_ID' + fields = ['source_OID', 'tariff_ID'] + else: + id_field_name = 'tariff_ID' + mapping_id_field_name = 'MASTER_OID' + fields = ['tariff_ID', 'MASTER_OID'] + + # do the search cursor: + logger.debug("start: search cursor on: fc: {} with fields: {}".format(fc, fields)) + with arcpy.da.SearchCursor(os.path.join(the_scenario.main_gdb, fc), fields) as cursor: + for row in cursor: + + id = row[0] + mapping_id = row[1] + capacity_update_list.append([fc, id_field_name, id, mapping_id_field_name, mapping_id]) + + logger.debug("start: execute many on: fc: {} with len: {}".format(fc, len(capacity_update_list))) + + if len(capacity_update_list) > 0: + sql = "insert or ignore into pipeline_mapping " \ + "(source, " \ + "id_field_name, " \ + "id, " \ + "mapping_id_field_name, " \ + "mapping_id) " \ + "values (?, ?, ?, ?, ?);" + + main_db_con.executemany(sql, capacity_update_list) + main_db_con.commit() + + +# ============================================================================== + +def locations_add_links(logger, the_scenario, modal_layer_name, max_artificial_link_distance_miles): + + # ADD LINKS LOGIC + # first we near the mode to the locations fc + # then we iterate through the near table and build up a dictionary of links and all the near XYs on that link. + # then we split the links on the mode (except pipeline) and preserve the data of that link. + # then we near the locations to the nodes on the now split links. + # we ignore locations with near dist == 0 on those nodes. + # then we add the artificial link and note which locations got links. + # then we set the connects_to field if the location was connected. + + logger.debug("start: locations_add_links for mode: {}".format(modal_layer_name)) + + scenario_gdb = the_scenario.main_gdb + fp_to_modal_layer = os.path.join(scenario_gdb, "network", modal_layer_name) + + locations_fc = the_scenario.locations_fc + arcpy.DeleteField_management(fp_to_modal_layer, "LOCATION_ID") + arcpy.AddField_management(os.path.join(scenario_gdb, modal_layer_name), "LOCATION_ID", "long") + + arcpy.DeleteField_management(fp_to_modal_layer, "LOCATION_ID_NAME") + arcpy.AddField_management(os.path.join(scenario_gdb, modal_layer_name), "LOCATION_ID_NAME", "text") + + if float(max_artificial_link_distance_miles.strip(" Miles")) < 0.0000001: + logger.warning("Note: ignoring mode {}. User specified artificial link distance of {}".format( + modal_layer_name, max_artificial_link_distance_miles)) + logger.debug("Setting the definition query to artificial = 99999, so we get an empty dataset for the " + "make_feature_layer and subsequent near analysis") + + definition_query = "Artificial = 999999" # something to return an empty set + else: + definition_query = "Artificial = 0" # the normal def query. + + if "pipeline" in modal_layer_name: + + if arcpy.Exists(os.path.join(scenario_gdb, "network", fp_to_modal_layer + "_points")): + arcpy.Delete_management(os.path.join(scenario_gdb, "network", fp_to_modal_layer + "_points")) + + # limit near to end points + if arcpy.CheckProduct("ArcInfo") == "Available": + arcpy.FeatureVerticesToPoints_management(in_features=fp_to_modal_layer, + out_feature_class=fp_to_modal_layer + "_points", + point_location="BOTH_ENDS") + else: + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified feature vertices " + "process will be run.") + arcpy.AddGeometryAttributes_management(fp_to_modal_layer, "LINE_START_MID_END") + arcpy.MakeXYEventLayer_management(fp_to_modal_layer, "START_X", "START_Y", + "modal_start_points_lyr", LCC_PROJ) + + arcpy.MakeXYEventLayer_management(fp_to_modal_layer, "END_X", "END_Y", + "modal_end_points_lyr", LCC_PROJ) + + # Due to tool design, must define the feature class location and name slightly differently (separating + # scenario gdb from feature class name. fp_to_modal_layer is identical to scenario_gdb + "network" + # + modal_layer_name + arcpy.FeatureClassToFeatureClass_conversion("modal_start_points_lyr", + scenario_gdb, + os.path.join("network", modal_layer_name + "_points")) + + arcpy.Append_management(["modal_end_points_lyr"], + fp_to_modal_layer + "_points", "NO_TEST") + + arcpy.Delete_management("modal_start_points_lyr") + arcpy.Delete_management("modal_end_points_lyr") + arcpy.DeleteField_management(fp_to_modal_layer, "START_X") + arcpy.DeleteField_management(fp_to_modal_layer, "START_Y") + arcpy.DeleteField_management(fp_to_modal_layer, "MID_X") + arcpy.DeleteField_management(fp_to_modal_layer, "MID_Y") + arcpy.DeleteField_management(fp_to_modal_layer, "END_X") + arcpy.DeleteField_management(fp_to_modal_layer, "END_Y") + + logger.debug("start: make_feature_layer_management") + arcpy.MakeFeatureLayer_management(fp_to_modal_layer + "_points", "modal_lyr_" + modal_layer_name, + definition_query) + + else: + logger.debug("start: make_feature_layer_management") + arcpy.MakeFeatureLayer_management(fp_to_modal_layer, "modal_lyr_" + modal_layer_name, definition_query) + + logger.debug("adding links between locations_fc and mode {} with max dist of {}".format(modal_layer_name, + max_artificial_link_distance_miles)) + + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near")): + logger.debug("start: delete tmp near") + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near")) + + logger.debug("start: generate_near") + if arcpy.CheckProduct("ArcInfo") == "Available": + arcpy.GenerateNearTable_analysis(locations_fc, "modal_lyr_" + modal_layer_name, + os.path.join(scenario_gdb, "tmp_near"), + max_artificial_link_distance_miles, "LOCATION", "NO_ANGLE", "CLOSEST") + + else: + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified generate_near process " + "will be run.") + # Spatial Join + # Workaround for GenerateNearTable not being available for lower-level ArcGIS licenses. + + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_spatial_join")): + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join")) + + # First, add field to capture joined FID + arcpy.AddField_management("modal_lyr_" + modal_layer_name, "Join_FID", "LONG") + arcpy.CalculateField_management("modal_lyr_" + modal_layer_name, "Join_FID", "!OBJECTID!", + "PYTHON_9.3") + + arcpy.SpatialJoin_analysis(locations_fc, "modal_lyr_" + modal_layer_name, + os.path.join(scenario_gdb, "tmp_spatial_join"), + match_option="CLOSEST", search_radius=max_artificial_link_distance_miles) + + arcpy.DeleteField_management("modal_lyr_" + modal_layer_name, "Join_FID") + + # queryPointAndDistance on the original point and corresponding spatial join match + # For line in spatial_join: + result_dict = {} + + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_spatial_join"), + ["Target_FID", "Join_FID", "SHAPE@"]) as scursor1: + for row1 in scursor1: + with arcpy.da.SearchCursor("modal_lyr_" + modal_layer_name, + ["OBJECTID", "SHAPE@"]) as scursor2: + for row2 in scursor2: + if row1[1] == row2[0]: + if "pipeline" in modal_layer_name: + result = row2[1].angleAndDistanceTo(row1[2], "PLANAR") + # Capture the point geometry of the nearest point on the polyline to the location point + # and the minimum distance between the line and the point + # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist + result_dict[row1[0]] = [row1[1], row1[2], row2[1], result[1]] + else: + result = row2[1].queryPointAndDistance(row1[2], False) + # Capture the point geometry of the nearest point on the polyline to the location point + # and the minimum distance between the line and the point + # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist + result_dict[row1[0]] = [row1[1], row1[2], result[0], result[2]] + + # Write to a tmp_near table equivalent to what is create by Generate Near Table tool + arcpy.CreateTable_management(scenario_gdb, "tmp_near") + arcpy.AddField_management("tmp_near", "IN_FID", "LONG") + arcpy.AddField_management("tmp_near", "NEAR_FID", "LONG") + arcpy.AddField_management("tmp_near", "NEAR_DIST", "LONG") + arcpy.AddField_management("tmp_near", "FROM_X", "DOUBLE") + arcpy.AddField_management("tmp_near", "FROM_Y", "DOUBLE") + arcpy.AddField_management("tmp_near", "NEAR_X", "DOUBLE") + arcpy.AddField_management("tmp_near", "NEAR_Y", "DOUBLE") + + # insert the relevant data into the table + icursor = arcpy.da.InsertCursor("tmp_near", ['IN_FID', 'NEAR_FID', 'NEAR_DIST', 'FROM_X', 'FROM_Y', + 'NEAR_X', 'NEAR_Y']) + + for in_fid in result_dict: + near_fid = result_dict[in_fid][0] + near_distance = result_dict[in_fid][3] + from_x = result_dict[in_fid][1].firstPoint.X + from_y = result_dict[in_fid][1].firstPoint.Y + near_x = result_dict[in_fid][2].firstPoint.X + near_y = result_dict[in_fid][2].firstPoint.Y + icursor.insertRow([in_fid, near_fid, near_distance, from_x, from_y, near_x, near_y]) + + del icursor + + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join")) + + edit = arcpy.da.Editor(os.path.join(scenario_gdb)) + edit.startEditing(False, False) + edit.startOperation() + + id_fieldname = arcpy.Describe(os.path.join(scenario_gdb, modal_layer_name)).OIDFieldName + + seenids = {} + + # SPLIT LINKS LOGIC + # 1) first search through the tmp_near fc and add points from the near on that link. + # 2) next we query the mode layer and get the mode specific data using the near FID. + # 3) then we split the old link, and use insert cursor to populate mode specific data into fc for the two new links. + # 4) then we delete the old unsplit link + logger.debug("start: split links") + + if arcpy.CheckProduct("ArcInfo") != "Available": + # Adding warning here rather than within the search cursor loop + logger.warning( + "The Advanced/ArcInfo license level of ArcGIS is not available. Modified split links process " + "will be run.") + + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_near"), + ["NEAR_FID", "NEAR_X", "NEAR_Y", "NEAR_DIST"]) as scursor: + + for row in scursor: + + # if the near distance is 0, then its connected and we don't need to split the line + if row[3] == 0: + # only give debug warning if not pipeline + if "pipeline" not in modal_layer_name: + logger.warning( + "Split links code: LOCATION MIGHT BE ON THE NETWORK. Ignoring NEAR_FID {} with NEAR_DIST {}".format( + row[0], row[3])) + + if not row[3] == 0: + + # STEP 1: point geoms where to split from the near XY + # --------------------------------------------------- + # get the line ID to split + theIdToGet = str(row[0]) # this is the link id we need + + if not theIdToGet in seenids: + seenids[theIdToGet] = [] + + point = arcpy.Point() + point.X = float(row[1]) + point.Y = float(row[2]) + point_geom = arcpy.PointGeometry(point, ftot_supporting_gis.LCC_PROJ) + seenids[theIdToGet].append(point_geom) + + # STEP 2 -- get mode specific data from the link + # ------------------------------------------------ + if 'pipeline' not in modal_layer_name: + + for theIdToGet in seenids: + + # initialize the variables so we dont get any gremlins + in_line = None # the shape geometry + in_capacity = None # road + rail + in_volume = None # road + rail + in_vcr = None # road + rail | volume to capacity ratio + in_fclass = None # road | fclass + in_speed = None # road | rounded speed + in_stracnet = None # rail + in_density_code = None # rail + in_tot_up_dwn = None # water + + if modal_layer_name == 'road': + for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), + ["SHAPE@", "Capacity", "Volume", "VCR", "FCLASS", "ROUNDED_SPEED"], + where_clause=id_fieldname + " = " + theIdToGet): + in_line = row[0] + in_capacity = row[1] + in_volume = row[2] + in_vcr = row[3] + in_fclass = row[4] + in_speed = row[5] + + if modal_layer_name == 'rail': + for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), + ["SHAPE@", "Capacity", "Volume", "VCR", "STRACNET", + "DENSITY_CODE"], where_clause=id_fieldname + " = " + theIdToGet): + in_line = row[0] + in_capacity = row[1] + in_volume = row[2] + in_vcr = row[3] + in_stracnet = row[4] + in_density_code = row[5] + + if modal_layer_name == 'water': + for row in arcpy.da.SearchCursor(os.path.join(scenario_gdb, modal_layer_name), + ["SHAPE@", "Capacity", "Volume", "VCR", "TOT_UP_DWN"], + where_clause=id_fieldname + " = " + theIdToGet): + in_line = row[0] + in_capacity = row[1] + in_volume = row[2] + in_vcr = row[3] + in_tot_up_dwn = row[4] + + # STEP 3: Split and populate with mode specific data from old link + # ------------------------------------------------------------------ + if arcpy.CheckProduct("ArcInfo") == "Available": + split_lines = arcpy.management.SplitLineAtPoint(in_line, seenids[theIdToGet], arcpy.Geometry(), 1) + + else: + # This is the alternative approach for those without an Advanced/ArcInfo license + point_list = seenids[theIdToGet] + line_list = [in_line] + split_lines = [] + continue_iteration = 'continue running' + + while continue_iteration == 'continue running': + line_list, point_list, split_lines, continue_iteration = cut_lines(line_list, point_list, split_lines) + + if not len(split_lines) == 1: + + # ROAD + if modal_layer_name == 'road': + + icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), + ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'FCLASS', + 'ROUNDED_SPEED', 'Volume', 'Capacity', 'VCR']) + + # Insert new links that include the mode-specific attributes + for new_line in split_lines: + len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude + icursor.insertRow( + [new_line, 0, modal_layer_name, len_in_miles, in_fclass, in_speed, in_volume, + in_capacity, in_vcr]) + + # Delete cursor object + del icursor + + elif modal_layer_name == 'rail': + icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), + ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'STRACNET', + 'DENSITY_CODE', 'Volume', 'Capacity', 'VCR']) + + # Insert new rows that include the mode-specific attributes + for new_line in split_lines: + len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude + icursor.insertRow( + [new_line, 0, modal_layer_name, len_in_miles, in_stracnet, in_density_code, in_volume, + in_capacity, in_vcr]) + + # Delete cursor object + del icursor + + elif modal_layer_name == 'water': + + icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), + ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'TOT_UP_DWN', + 'Volume', 'Capacity', 'VCR']) + + # Insert new rows that include the mode-specific attributes + for new_line in split_lines: + len_in_miles = Q_(new_line.length, "meters").to("miles").magnitude + icursor.insertRow( + [new_line, 0, modal_layer_name, len_in_miles, in_tot_up_dwn, in_volume, in_capacity, + in_vcr]) + + # Delete cursor object + del icursor + + else: + logger.warning("Modal_layer_name: {} is not supported.".format(modal_layer_name)) + + # STEP 4: Delete old unsplit data + with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, modal_layer_name), ['OID@'], + where_clause=id_fieldname + " = " + theIdToGet) as ucursor: + for row in ucursor: + ucursor.deleteRow() + + # if the split doesn't work + else: + logger.detailed_debug( + "the line split didn't work for ID: {}. " + "Might want to investigate. " + "Could just be an artifact from the near result being the end of a line.".format( + theIdToGet)) + + edit.stopOperation() + edit.stopEditing(True) + + # delete the old features + # ------------------------ + logger.debug("start: delete old features (tmp_near, tmp_near_2, tmp_nodes)") + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near")): + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near")) + + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_near_2")): + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near_2")) + + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_nodes")): + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_nodes")) + + # add artificial links + # now that the lines have been split add lines from the from points to the nearest node + # -------------------------------------------------------------------------------------- + logger.debug("start: add artificial links now w/ definition_query: {}".format(definition_query)) + logger.debug("start: make_featurelayer 2") + fp_to_modal_layer = os.path.join(scenario_gdb, "network", modal_layer_name) + arcpy.MakeFeatureLayer_management(fp_to_modal_layer, "modal_lyr_" + modal_layer_name + "2", definition_query) + logger.debug("start: feature vertices to points 2") + if arcpy.CheckProduct("ArcInfo") == "Available": + arcpy.FeatureVerticesToPoints_management(in_features="modal_lyr_" + modal_layer_name + "2", + out_feature_class=os.path.join(scenario_gdb, "tmp_nodes"), + point_location="BOTH_ENDS") + else: + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified feature vertices " + "process will be run.") + arcpy.AddGeometryAttributes_management("modal_lyr_" + modal_layer_name + "2", "LINE_START_MID_END") + arcpy.MakeXYEventLayer_management("modal_lyr_" + modal_layer_name + "2", "START_X", "START_Y", + "modal_start_points_lyr", LCC_PROJ) + + arcpy.MakeXYEventLayer_management("modal_lyr_" + modal_layer_name + "2", "END_X", "END_Y", + "modal_end_points_lyr", LCC_PROJ) + + arcpy.FeatureClassToFeatureClass_conversion("modal_start_points_lyr", + scenario_gdb, + "tmp_nodes") + + arcpy.Append_management(["modal_end_points_lyr"], + "tmp_nodes", "NO_TEST") + + arcpy.Delete_management("modal_start_points_lyr") + arcpy.Delete_management("modal_end_points_lyr") + arcpy.DeleteField_management(fp_to_modal_layer, "START_X") + arcpy.DeleteField_management(fp_to_modal_layer, "START_Y") + arcpy.DeleteField_management(fp_to_modal_layer, "MID_X") + arcpy.DeleteField_management(fp_to_modal_layer, "MID_Y") + arcpy.DeleteField_management(fp_to_modal_layer, "END_X") + arcpy.DeleteField_management(fp_to_modal_layer, "END_Y") + + logger.debug("start: generate near table 2") + if arcpy.CheckProduct("ArcInfo") == "Available": + arcpy.GenerateNearTable_analysis(locations_fc, os.path.join(scenario_gdb, "tmp_nodes"), + os.path.join(scenario_gdb, "tmp_near_2"), + max_artificial_link_distance_miles, "LOCATION", "NO_ANGLE", "CLOSEST") + + else: + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. Modified generate_near process " + "will be run.") + # Spatial Join + # Workaround for GenerateNearTable not being available for lower-level ArcGIS licenses + + if arcpy.Exists(os.path.join(scenario_gdb, "tmp_spatial_join_2")): + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join_2")) + + # First, add field to capture joined FID + arcpy.AddField_management("tmp_nodes", "Join_FID", "LONG") + arcpy.CalculateField_management("tmp_nodes", "Join_FID", "!OBJECTID!", "PYTHON_9.3") + + arcpy.SpatialJoin_analysis(locations_fc, "tmp_nodes", + os.path.join(scenario_gdb, "tmp_spatial_join_2"), + match_option="CLOSEST", search_radius=max_artificial_link_distance_miles) + + # queryPointAndDistance on the original point and corresponding spatial join match + # For line in spatial_join: + result_dict = {} + + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_spatial_join_2"), + ["Target_FID", "Join_FID", "SHAPE@"]) as scursor1: + for row1 in scursor1: + with arcpy.da.SearchCursor("tmp_nodes", + ["OBJECTID", "SHAPE@"]) as scursor2: + for row2 in scursor2: + if row1[1] == row2[0]: + result = row2[1].angleAndDistanceTo(row1[2], "PLANAR") + # Capture the point geometry of the nearest point on the polyline to the location point + # and the minimum distance between the line and the point + # result_dict[in_fid] = [near_fid, from_xy, near_xy, near_dist + result_dict[row1[0]] = [row1[1], row1[2], row2[1], result[1]] + + # Write to a tmp_near table equivalent to what is create by Generate Near Table tool + arcpy.CreateTable_management(scenario_gdb, "tmp_near_2") + arcpy.AddField_management("tmp_near_2", "IN_FID", "LONG") + arcpy.AddField_management("tmp_near_2", "NEAR_FID", "LONG") + arcpy.AddField_management("tmp_near_2", "NEAR_DIST", "LONG") + arcpy.AddField_management("tmp_near_2", "FROM_X", "DOUBLE") + arcpy.AddField_management("tmp_near_2", "FROM_Y", "DOUBLE") + arcpy.AddField_management("tmp_near_2", "NEAR_X", "DOUBLE") + arcpy.AddField_management("tmp_near_2", "NEAR_Y", "DOUBLE") + + # insert the relevant data into the table + icursor = arcpy.da.InsertCursor("tmp_near_2", ['IN_FID', 'NEAR_FID', 'NEAR_DIST', 'FROM_X', 'FROM_Y', + 'NEAR_X', 'NEAR_Y']) + + for in_fid in result_dict: + near_fid = result_dict[in_fid][0] + near_distance = result_dict[in_fid][3] + from_x = result_dict[in_fid][1].firstPoint.X + from_y = result_dict[in_fid][1].firstPoint.Y + near_x = result_dict[in_fid][2].firstPoint.X + near_y = result_dict[in_fid][2].firstPoint.Y + icursor.insertRow([in_fid, near_fid, near_distance, from_x, from_y, near_x, near_y]) + + del icursor + + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_spatial_join_2")) + + logger.debug("start: delete tmp_nodes") + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_nodes")) + + logger.debug("start: start editor") + edit = arcpy.da.Editor(os.path.join(scenario_gdb)) + edit.startEditing(False, False) + edit.startOperation() + + icursor = arcpy.da.InsertCursor(os.path.join(scenario_gdb, modal_layer_name), + ['SHAPE@', 'Artificial', 'MODE_TYPE', 'MILES', 'LOCATION_ID', + 'LOCATION_ID_NAME']) # add location_id for setting flow restrictions + + location_id_name_dict = get_location_id_name_dict(the_scenario, logger) + connected_location_ids = [] + connected_location_id_names = [] + logger.debug("start: search cursor on tmp_near_2") + with arcpy.da.SearchCursor(os.path.join(scenario_gdb, "tmp_near_2"), + ["FROM_X", "FROM_Y", "NEAR_X", "NEAR_Y", "NEAR_DIST", "IN_FID"]) as scursor: + + for row in scursor: + + if not row[4] == 0: + + # use the unique objectid (in_fid) from the near to determine + # if we have an in or an out location. + # then set the flow restrictions appropriately. + + in_fid = row[5] + location_id_name = location_id_name_dict[in_fid] + location_id = location_id_name.split("_")[0] + connected_location_ids.append(location_id) + connected_location_id_names.append(location_id_name) + + coordList = [] + coordList.append(arcpy.Point(row[0], row[1])) + coordList.append(arcpy.Point(row[2], row[3])) + polyline = arcpy.Polyline(arcpy.Array(coordList)) + + len_in_miles = Q_(polyline.length, "meters").to("miles").magnitude + + # insert artificial link attributes + icursor.insertRow([polyline, 1, modal_layer_name, len_in_miles, location_id, location_id_name]) + + else: + logger.warning("Artificial Link code: Ignoring NEAR_FID {} with NEAR_DIST {}".format(row[0], row[4])) + + del icursor + logger.debug("start: stop editing") + edit.stopOperation() + edit.stopEditing(True) + + arcpy.Delete_management(os.path.join(scenario_gdb, "tmp_near_2")) + + # ALSO SET CONNECTS_X FIELD IN POINT LAYER + # ----------------------------------------- + logger.debug("start: connect_x") + + edit = arcpy.da.Editor(scenario_gdb) + edit.startEditing(False, False) + edit.startOperation() + with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, locations_fc), + ["LOCATION_ID_NAME", "connects_" + modal_layer_name]) as cursor: + + for row in cursor: + + if row[0] in connected_location_id_names: + row[1] = 1 + cursor.updateRow(row) + + edit.stopOperation() + edit.stopEditing(True) + + logger.debug("finish: locations_add_links") + + +# ============================================================================== + + +def ignore_locations_not_connected_to_network(the_scenario, logger): + logger.info("start: ignore_locations_not_connected_to_network") + logger.debug("flag locations which don't connect to the network") + + # flag locations which dont connect to the network in the GIS + # ----------------------------------------------------------- + + # add the ignore field to the fc + scenario_gdb = the_scenario.main_gdb + locations_fc = the_scenario.locations_fc + + edit = arcpy.da.Editor(scenario_gdb) + edit.startEditing(False, False) + edit.startOperation() + + query = "connects_road = 0 and " \ + "connects_rail = 0 and " \ + "connects_water = 0 and " \ + "connects_pipeline_prod_trf_rts = 0 and " \ + "connects_pipeline_crude_trf_rts = 0" + + list_of_ignored_locations = [] + with arcpy.da.UpdateCursor(locations_fc, ['location_id', 'ignore'], where_clause=query) as ucursor: + for row in ucursor: + list_of_ignored_locations.append(row[0]) + logger.debug('Location ID {} does not connect to the network and will be ignored'.format(row[0])) + row[1] = 1 + ucursor.updateRow(row) + + if len(list_of_ignored_locations) > 0: + + logger.result("# of locations not connected to network and ignored: \t{}".format(len( + list_of_ignored_locations)/2.0)) + logger.info("note: check the log files for additional debug information.") + + edit.stopOperation() + edit.stopEditing(True) + + # flag locations which dont connect to the network in the DB + # ----------------------------------------------------------- + scenario_db = the_scenario.main_db + + if os.path.exists(scenario_db): + + with sqlite3.connect(scenario_db) as db_con: + + logger.debug("connected to the db") + + db_cur = db_con.cursor() + + # iterate through the locations table and set ignore flag + # if location_id is in the list_of_ignored_facilities + # ------------------------------------------------------------ + + logger.debug("setting ignore fields for location in the DB locations and facilities tables") + + sql = "select location_id, ignore_location from locations;" + + db_cur.execute(sql) + for row in db_cur: + + if str(row[0]) in list_of_ignored_locations: # cast as string since location_id field is a string + ignore_flag = 'network' + else: + ignore_flag = 'false' + sql = "update locations set ignore_location = '{}' where location_id = '{}';".format(ignore_flag, + row[0]) + db_con.execute(sql) + + # iterate through the facilities table, + # and check if location_id is in the list_of_ignored_facilities + # ------------------------------------------------------------ + sql = "update facilities set ignore_facility = '{}' where location_id = '{}';".format(ignore_flag, + row[0]) + + db_con.execute(sql) + + logger.debug("finished: ignore_locations_not_connected_to_network") + return 1 + +# ====================================================================================================================== + +# this code subsets the road network so that only links within the minimum bounding geometry (MBG) buffer are +# preserved. This method ends with the arcpy.Compact_management() call that reduces the size of the geodatabase and +# increases overall performance when hooking facilities into the network, and exporting the road fc to a shapefile +# for networkX. + + +def minimum_bounding_geometry(the_scenario, logger): + logger.info("start: minimum_bounding_geometry") + arcpy.env.workspace = the_scenario.main_gdb + + # Clean up any left of layers from a previous run + if arcpy.Exists("road_lyr"): + arcpy.Delete_management("road_lyr") + if arcpy.Exists("rail_lyr"): + arcpy.Delete_management("rail_lyr") + if arcpy.Exists("water_lyr"): + arcpy.Delete_management("water_lyr") + if arcpy.Exists("pipeline_prod_trf_rts_lyr"): + arcpy.Delete_management("pipeline_prod_trf_rts_lyr") + if arcpy.Exists("pipeline_crude_trf_rts_lyr"): + arcpy.Delete_management("pipeline_crude_trf_rts_lyr") + if arcpy.Exists("Locations_MBG"): + arcpy.Delete_management("Locations_MBG") + if arcpy.Exists("Locations_MBG_Buffered"): + arcpy.Delete_management("Locations_MBG_Buffered") + + # Determine the minimum bounding geometry of the scenario + # The advanced license is required to use the convex hull method. If not available, default to rectangle_by_area + # which will not subset things quite as small but is still better than no subsetting at all + if arcpy.CheckProduct("ArcInfo") == "Available": + arcpy.MinimumBoundingGeometry_management("Locations", "Locations_MBG", "CONVEX_HULL") + + else: + arcpy.MinimumBoundingGeometry_management("Locations", "Locations_MBG", "RECTANGLE_BY_AREA") + logger.warning("The Advanced/ArcInfo license level of ArcGIS is not available. A slight modification to " + "the minimum bounding geometry process is necessary to ensure FTOT can successfully run.") + + # Buffer the minimum bounding geometry of the scenario + arcpy.Buffer_analysis("Locations_MBG", "Locations_MBG_Buffered", "100 Miles", "FULL", "ROUND", "NONE", "", + "GEODESIC") + + # Select the roads within the buffer + # ----------------------------------- + arcpy.MakeFeatureLayer_management("road", "road_lyr") + arcpy.SelectLayerByLocation_management("road_lyr", "INTERSECT", "Locations_MBG_Buffered") + + result = arcpy.GetCount_management("road") + count_all_roads = float(result.getOutput(0)) + + result = arcpy.GetCount_management("road_lyr") + count_roads_subset = float(result.getOutput(0)) + + ## CHANGE + if count_all_roads > 0: + roads_percentage = count_roads_subset / count_all_roads + else: + roads_percentage = 0 + + # Only subset if the subset will result in substantial reduction of the road network size + if roads_percentage < 0.75: + # Switch selection to identify what's outside the buffer + arcpy.SelectLayerByAttribute_management("road_lyr", "SWITCH_SELECTION") + + # Add in FC 1 roadways (going to keep all interstate highways) + arcpy.SelectLayerByAttribute_management("road_lyr", "REMOVE_FROM_SELECTION", "FCLASS = 1") + + # Delete the features outside the buffer + with arcpy.da.UpdateCursor('road_lyr', ['OBJECTID']) as ucursor: + for ucursor_row in ucursor: + ucursor.deleteRow() + + arcpy.Delete_management("road_lyr") + + # # Select the rail within the buffer + # # --------------------------------- + + arcpy.Delete_management("Locations_MBG") + arcpy.Delete_management("Locations_MBG_Buffered") + + # finally, compact the geodatabase so the MBG has an effect on runtime. + arcpy.Compact_management(the_scenario.main_gdb) + logger.debug("finish: minimum_bounding_geometry") + diff --git a/program/ftot_scenario.py b/program/ftot_scenario.py index 5a0a9d7..7e80dcb 100644 --- a/program/ftot_scenario.py +++ b/program/ftot_scenario.py @@ -1,664 +1,681 @@ - -#--------------------------------------------------------------------------------------------------- -# Name: ftot_scenario -# -# Purpose: declare all of the attributes of the_scenario object. -# create getter and setter methods for each attribute. -# -#--------------------------------------------------------------------------------------------------- - - - -import os -import sys -from xml.dom import minidom -from ftot import SCHEMA_VERSION -from ftot import Q_, ureg -import sqlite3 - -try: - from lxml import etree -except ImportError: - print ("This script requires the lxml Python library to validate the XML scenario file.") - print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") - print("Exiting...") - sys.exit() - -#=================================================================================================== - -def getElementFromXmlFile(xmlFile, elementName): - return xmlFile.getElementsByTagName(elementName)[0].firstChild.data - -#=================================================================================================== - -def format_number(numString): - """Removes any number formatting, i.e., thousand's separator or dollar sign""" - - if numString.rfind(",") > -1: - numString = numString.replace(",", "") - if numString.rfind("$") > -1: - numString = numString.replace("$", "") - return float(numString) - -#=================================================================================================== - -class Scenario: - def __init__(self): - pass - -def load_scenario_config_file(fullPathToXmlConfigFile, fullPathToXmlSchemaFile, logger): - - if not os.path.exists(fullPathToXmlConfigFile): - raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) - - if fullPathToXmlConfigFile.rfind(".xml") < 0: - raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) - - if not os.path.exists(fullPathToXmlSchemaFile): - raise IOError("XML Schema File not found at {}".format(fullPathToXmlSchemaFile)) - - xmlScenarioFile = minidom.parse(fullPathToXmlConfigFile) - - # Validate XML scenario against XML schema - schemaObj = etree.XMLSchema(etree.parse(fullPathToXmlSchemaFile)) - xmlFileObj = etree.parse(fullPathToXmlConfigFile) - validationResult = schemaObj.validate(xmlFileObj) - - logger.debug("validate XML scenario against XML schema") - if validationResult == False: - logger.warning("XML scenario validation failed. Error messages to follow.") - for error in schemaObj.error_log: - logger.warning("ERROR ON LINE: {} - ERROR MESSAGE: {}".format(error.line, error.message)) - - raise Exception("XML Scenario File does not meet the requirements in the XML schema file.") - - # initialize scenario ojbect - logger.debug("initialize scenario object") - scenario = Scenario() - - # Check the scenario schema version (current version is specified in FTOT.py under VERSION_NUMBER global var) - logger.debug("validate schema version is correct") - scenario.scenario_schema_version = xmlScenarioFile.getElementsByTagName('Scenario_Schema_Version')[0].firstChild.data - - #if not str(VERSION_NUMBER) == str(scenario.scenario_schema_version): - if not str(SCHEMA_VERSION).split(".")[0:2] == str(scenario.scenario_schema_version).split(".")[0:2]: - error = "XML Schema File Version is {}. Expected version {}. " \ - "Use the XML flag to run the XML upgrade tool. " \ - .format(str(scenario.scenario_schema_version), str(SCHEMA_VERSION)) - logger.error(error) - raise Exception(error) - - scenario.scenario_name = xmlScenarioFile.getElementsByTagName('Scenario_Name')[0].firstChild.data - # Convert any commas in the scenario name to dashes - warning = "Replace any commas in the scenario name with dashes to accomodate csv files." - logger.debug(warning) - scenario.scenario_name = scenario.scenario_name.replace(",", "-") - scenario.scenario_description = xmlScenarioFile.getElementsByTagName('Scenario_Description')[0].firstChild.data - - # SCENARIO INPUTS SECTION - # ---------------------------------------------------------------------------------------- - scenario.common_data_folder = xmlScenarioFile.getElementsByTagName('Common_Data_Folder')[0].firstChild.data - scenario.base_network_gdb = xmlScenarioFile.getElementsByTagName('Base_Network_Gdb')[0].firstChild.data - scenario.disruption_data = xmlScenarioFile.getElementsByTagName('Disruption_Data')[0].firstChild.data - scenario.base_rmp_layer = xmlScenarioFile.getElementsByTagName('Base_RMP_Layer')[0].firstChild.data - scenario.base_destination_layer = xmlScenarioFile.getElementsByTagName('Base_Destination_Layer')[0].firstChild.data - scenario.base_processors_layer = xmlScenarioFile.getElementsByTagName('Base_Processors_Layer')[0].firstChild.data - - scenario.rmp_commodity_data = xmlScenarioFile.getElementsByTagName('RMP_Commodity_Data')[0].firstChild.data - scenario.destinations_commodity_data = xmlScenarioFile.getElementsByTagName('Destinations_Commodity_Data')[0].firstChild.data - scenario.processors_commodity_data = xmlScenarioFile.getElementsByTagName('Processors_Commodity_Data')[0].firstChild.data - scenario.processors_candidate_slate_data = xmlScenarioFile.getElementsByTagName('Processors_Candidate_Commodity_Data')[0].firstChild.data - # note: the processor_candidates_data is defined under other since it is not a user specified file. - scenario.schedule = xmlScenarioFile.getElementsByTagName('Schedule_Data')[0].firstChild.data - scenario.commodity_mode_data = xmlScenarioFile.getElementsByTagName('Commodity_Mode_Data')[0].firstChild.data - - # use pint to set the default units - logger.debug("test: setting the default units with pint") - try: - scenario.default_units_solid_phase = Q_(xmlScenarioFile.getElementsByTagName('Default_Units_Solid_Phase')[0].firstChild.data).units - scenario.default_units_liquid_phase = Q_(xmlScenarioFile.getElementsByTagName('Default_Units_Liquid_Phase')[0].firstChild.data).units - logger.debug("PASS: setting the default units with pint") - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # ASSUMPTIONS SECTION - # ---------------------------------------------------------------------------------------- - - # solid and liquid vehicle loads - try: - - logger.debug("test: setting the vehicle loads for solid phase of matter with pint") - scenario.truck_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Truck_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) - scenario.railcar_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Railcar_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) - scenario.barge_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Barge_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) - - logger.debug("test: setting the vehicle loads for liquid phase of matter with pint") - scenario.truck_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Truck_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) - scenario.railcar_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Railcar_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) - scenario.barge_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Barge_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) - scenario.pipeline_crude_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Crude_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) - scenario.pipeline_prod_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Prod_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) - logger.debug("PASS: setting the vehicle loads with pint passed") - - logger.debug("test: setting the vehicle fuel efficiencies with pint") - scenario.truckFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Truck_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') - scenario.railFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Rail_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') - scenario.bargeFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Barge_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') - logger.debug("PASS: setting the vehicle fuel efficiencies with pint passed") - - logger.debug("test: setting the vehicle emission factors with pint") - scenario.CO2urbanUnrestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Urban_Unrestricted')[0].firstChild.data).to('g/mi') - scenario.CO2urbanRestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Urban_Restricted')[0].firstChild.data).to('g/mi') - scenario.CO2ruralUnrestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Rural_Unrestricted')[0].firstChild.data).to('g/mi') - scenario.CO2ruralRestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Rural_Restricted')[0].firstChild.data).to('g/mi') - scenario.railroadCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Railroad_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) - scenario.bargeCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Barge_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) - scenario.pipelineCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) - logger.debug("PASS: setting the vehicle emission factors with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - - - # SCRIPT PARAMETERS SECTION FOR NETWORK - # ---------------------------------------------------------------------------------------- - - # rail costs - try: - logger.debug("test: setting the base costs for rail with pint") - scenario.solid_railroad_class_1_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Railroad_Class_I_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) - scenario.liquid_railroad_class_1_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Railroad_Class_I_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) - logger.debug("PASS: setting the base costs for rail with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # rail penalties - scenario.rail_dc_7 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_7_Weight')[0].firstChild.data) - scenario.rail_dc_6 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_6_Weight')[0].firstChild.data) - scenario.rail_dc_5 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_5_Weight')[0].firstChild.data) - scenario.rail_dc_4 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_4_Weight')[0].firstChild.data) - scenario.rail_dc_3 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_3_Weight')[0].firstChild.data) - scenario.rail_dc_2 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_2_Weight')[0].firstChild.data) - scenario.rail_dc_1 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_1_Weight')[0].firstChild.data) - scenario.rail_dc_0 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_0_Weight')[0].firstChild.data) - - # truck costs - try: - logger.debug("test: setting the base costs for truck with pint") - scenario.solid_truck_base_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Truck_Base_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) - scenario.liquid_truck_base_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Truck_Base_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) - logger.debug("PASS: setting the base costs for truck with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # road penalties - scenario.truck_interstate = format_number(xmlScenarioFile.getElementsByTagName('Truck_Interstate_Weight')[0].firstChild.data) - scenario.truck_pr_art = format_number(xmlScenarioFile.getElementsByTagName('Truck_Principal_Arterial_Weight')[0].firstChild.data) - scenario.truck_m_art = format_number(xmlScenarioFile.getElementsByTagName('Truck_Minor_Arterial_Weight')[0].firstChild.data) - scenario.truck_local = format_number(xmlScenarioFile.getElementsByTagName('Truck_Local_Weight')[0].firstChild.data) - - # barge costs - try: - logger.debug("test: setting the base costs for barge with pint") - scenario.solid_barge_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Barge_cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) - scenario.liquid_barge_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Barge_cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) - logger.debug("PASS: setting the base costs for barge with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # water penalties - scenario.water_high_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_High_Volume_Weight')[0].firstChild.data) - scenario.water_med_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_Medium_Volume_Weight')[0].firstChild.data) - scenario.water_low_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_Low_Volume_Weight')[0].firstChild.data) - scenario.water_no_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_No_Volume_Weight')[0].firstChild.data) - - # transloading costs - try: - logger.debug("test: setting the transloading costs with pint") - scenario.solid_transloading_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Transloading_Cost')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) - scenario.liquid_transloading_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Transloading_Cost')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) - logger.debug("PASS: setting the transloading costs with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # artificial link distances - try: - logger.debug("test: setting the artificial link distances with pint") - scenario.road_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Road_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') - scenario.rail_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Rail_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') - scenario.water_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Water_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') - scenario.pipeline_crude_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Crude_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') - scenario.pipeline_prod_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Products_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') - logger.debug("PASS: setting the artificial link distances with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # short haul penalties - try: - logger.debug("test: setting the short haul penalties with pint") - scenario.liquid_rail_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('liquid_Rail_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) - scenario.solid_rail_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('solid_Rail_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) - scenario.liquid_water_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('liquid_Water_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) - scenario.solid_water_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('solid_Water_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) - logger.debug("PASS: setting the short haul penalties with pint passed") - - except Exception as e: - logger.error("FAIL: {} ".format(e)) - raise Exception("FAIL: {}".format(e)) - - # RUN ROUTE OPTIMIZATION SCRIPT SECTION - # ---------------------------------------------------------------------------------------- - - # Setting flag for network density reduction based on 'NDR_On' field - if xmlScenarioFile.getElementsByTagName('NDR_On')[0].firstChild.data == "True": - scenario.ndrOn = True - else: - scenario.ndrOn = False - - scenario.permittedModes = [] - if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Road')[0].firstChild.data == "True": - scenario.permittedModes.append("road") - if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Rail')[0].firstChild.data == "True": - scenario.permittedModes.append("rail") - if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Water')[0].firstChild.data == "True": - scenario.permittedModes.append("water") - - if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Pipeline_Crude')[0].firstChild.data == "True": - scenario.permittedModes.append("pipeline_crude_trf_rts") - if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Pipeline_Prod')[0].firstChild.data == "True": - scenario.permittedModes.append("pipeline_prod_trf_rts") - - if xmlScenarioFile.getElementsByTagName('Capacity_On')[0].firstChild.data == "True": - scenario.capacityOn = True - else: - scenario.capacityOn = False - - scenario.backgroundFlowModes = [] - if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Road')[0].firstChild.data == "True": - scenario.backgroundFlowModes.append("road") - if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Rail')[0].firstChild.data == "True": - scenario.backgroundFlowModes.append("rail") - if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Water')[0].firstChild.data == "True": - scenario.backgroundFlowModes.append("water") - - if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Pipeline_Crude')[0].firstChild.data == "True": - scenario.backgroundFlowModes.append("pipeline") - if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Pipeline_Prod')[0].firstChild.data == "True": - scenario.backgroundFlowModes.append("pipeline") - - scenario.minCapacityLevel = float(xmlScenarioFile.getElementsByTagName('Minimum_Capacity_Level')[0].firstChild.data) - - scenario.unMetDemandPenalty = float(xmlScenarioFile.getElementsByTagName('Unmet_Demand_Penalty')[0].firstChild.data) - - # OTHER - # ---------------------------------------------------------------------------------------- - - scenario.scenario_run_directory = os.path.dirname(fullPathToXmlConfigFile) - - scenario.main_db = os.path.join(scenario.scenario_run_directory, "main.db") - scenario.main_gdb = os.path.join(scenario.scenario_run_directory, "main.gdb") - - scenario.rmp_fc = os.path.join(scenario.main_gdb, "raw_material_producers") - scenario.destinations_fc = os.path.join(scenario.main_gdb, "ultimate_destinations") - scenario.processors_fc = os.path.join(scenario.main_gdb, "processors") - scenario.processor_candidates_fc = os.path.join(scenario.main_gdb, "all_candidate_processors") - scenario.locations_fc = os.path.join(scenario.main_gdb, "locations") - - # this file is generated by the processor_candidates() method - scenario.processor_candidates_commodity_data = os.path.join(scenario.scenario_run_directory, "debug", "ftot_generated_processor_candidates.csv") - - # this is the directory to store the shp files that a programtically generated for the networkx read_shp method - scenario.networkx_files_dir = os.path.join(scenario.scenario_run_directory, "temp_networkx_shp_files") - - return scenario - -#=================================================================================================== - - -def dump_scenario_info_to_report(the_scenario, logger): - logger.config("xml_scenario_name: \t{}".format(the_scenario.scenario_name)) - logger.config("xml_scenario_description: \t{}".format(the_scenario.scenario_description)) - logger.config("xml_scenario_run_directory: \t{}".format(the_scenario.scenario_run_directory)) - logger.config("xml_scenario_schema_version: \t{}".format(the_scenario.scenario_schema_version)) - - logger.config("xml_common_data_folder: \t{}".format(the_scenario.common_data_folder)) - - logger.config("xml_base_network_gdb: \t{}".format(the_scenario.base_network_gdb)) - logger.config("xml_disruption_data: \t{}".format(the_scenario.disruption_data)) - - logger.config("xml_base_rmp_layer: \t{}".format(the_scenario.base_rmp_layer)) - logger.config("xml_base_destination_layer: \t{}".format(the_scenario.base_destination_layer)) - logger.config("xml_base_processors_layer: \t{}".format(the_scenario.base_processors_layer)) - - logger.config("xml_rmp_commodity_data: \t{}".format(the_scenario.rmp_commodity_data)) - logger.config("xml_destinations_commodity_data: \t{}".format(the_scenario.destinations_commodity_data)) - logger.config("xml_processors_commodity_data: \t{}".format(the_scenario.processors_commodity_data)) - logger.config("xml_processors_candidate_slate_data: \t{}".format(the_scenario.processors_candidate_slate_data)) - - logger.config("xml_schedule_data: \t{}".format(the_scenario.schedule)) - logger.config("xml_commodity_mode_data: \t{}".format(the_scenario.commodity_mode_data)) - - logger.config("xml_default_units_solid_phase: \t{}".format(the_scenario.default_units_solid_phase)) - logger.config("xml_default_units_liquid_phase: \t{}".format(the_scenario.default_units_liquid_phase)) - - logger.config("xml_truck_load_solid: \t{}".format(the_scenario.truck_load_solid)) - logger.config("xml_railcar_load_solid: \t{}".format(the_scenario.railcar_load_solid)) - logger.config("xml_barge_load_solid: \t{}".format(the_scenario.barge_load_solid)) - - logger.config("xml_truck_load_liquid: \t{}".format(the_scenario.truck_load_liquid)) - logger.config("xml_railcar_load_liquid: \t{}".format(the_scenario.railcar_load_liquid)) - logger.config("xml_barge_load_liquid: \t{}".format(the_scenario.barge_load_liquid)) - logger.config("xml_pipeline_crude_load_liquid: \t{}".format(the_scenario.pipeline_crude_load_liquid)) - logger.config("xml_pipeline_prod_load_liquid: \t{}".format(the_scenario.pipeline_prod_load_liquid)) - - logger.config("xml_solid_railroad_class_1_cost: \t{}".format(the_scenario.solid_railroad_class_1_cost)) - logger.config("xml_liquid_railroad_class_1_cost: \t{}".format(the_scenario.liquid_railroad_class_1_cost)) - logger.config("xml_rail_dc_7: \t{}".format(the_scenario.rail_dc_7)) - logger.config("xml_rail_dc_6: \t{}".format(the_scenario.rail_dc_6)) - logger.config("xml_rail_dc_5: \t{}".format(the_scenario.rail_dc_5)) - logger.config("xml_rail_dc_4: \t{}".format(the_scenario.rail_dc_4)) - logger.config("xml_rail_dc_3: \t{}".format(the_scenario.rail_dc_3)) - logger.config("xml_rail_dc_2: \t{}".format(the_scenario.rail_dc_2)) - logger.config("xml_rail_dc_1: \t{}".format(the_scenario.rail_dc_1)) - logger.config("xml_rail_dc_0: \t{}".format(the_scenario.rail_dc_0)) - - logger.config("xml_liquid_truck_base_cost: \t{}".format(the_scenario.liquid_truck_base_cost)) - logger.config("xml_solid_truck_base_cost: \t{}".format(the_scenario.solid_truck_base_cost)) - logger.config("xml_truck_interstate: \t{}".format(the_scenario.truck_interstate)) - logger.config("xml_truck_pr_art: \t{}".format(the_scenario.truck_pr_art)) - logger.config("xml_truck_m_art: \t{}".format(the_scenario.truck_m_art)) - logger.config("xml_truck_local: \t{}".format(the_scenario.truck_local)) - - logger.config("xml_liquid_barge_cost: \t{}".format(the_scenario.liquid_barge_cost)) - logger.config("xml_solid_barge_cost: \t{}".format(the_scenario.solid_barge_cost)) - logger.config("xml_water_high_vol: \t{}".format(the_scenario.water_high_vol)) - logger.config("xml_water_med_vol: \t{}".format(the_scenario.water_med_vol)) - logger.config("xml_water_low_vol: \t{}".format(the_scenario.water_low_vol)) - logger.config("xml_water_no_vol: \t{}".format(the_scenario.water_no_vol)) - - logger.config("xml_solid_transloading_cost: \t{}".format(the_scenario.solid_transloading_cost)) - logger.config("xml_liquid_transloading_cost: \t{}".format(the_scenario.liquid_transloading_cost)) - - logger.config("xml_road_max_artificial_link_dist: \t{}".format(the_scenario.road_max_artificial_link_dist)) - logger.config("xml_rail_max_artificial_link_dist: \t{}".format(the_scenario.rail_max_artificial_link_dist)) - logger.config("xml_water_max_artificial_link_dist: \t{}".format(the_scenario.water_max_artificial_link_dist)) - logger.config("xml_pipeline_crude_max_artificial_link_dist: \t{}".format(the_scenario.pipeline_crude_max_artificial_link_dist)) - logger.config("xml_pipeline_prod_max_artificial_link_dist: \t{}".format(the_scenario.pipeline_prod_max_artificial_link_dist)) - - logger.config("xml_liquid_rail_short_haul_penalty: \t{}".format(the_scenario.liquid_rail_short_haul_penalty)) - logger.config("xml_solid_rail_short_haul_penalty: \t{}".format(the_scenario.solid_rail_short_haul_penalty)) - logger.config("xml_liquid_water_short_haul_penalty: \t{}".format(the_scenario.liquid_water_short_haul_penalty)) - logger.config("xml_solid_water_short_haul_penalty: \t{}".format(the_scenario.solid_water_short_haul_penalty)) - - logger.config("xml_truckFuelEfficiency: \t{}".format(the_scenario.truckFuelEfficiency)) - logger.config("xml_bargeFuelEfficiency: \t{}".format(the_scenario.bargeFuelEfficiency)) - logger.config("xml_railFuelEfficiency: \t{}".format(the_scenario.railFuelEfficiency)) - logger.config("xml_CO2urbanUnrestricted: \t{}".format(the_scenario.CO2urbanUnrestricted)) - logger.config("xml_CO2urbanRestricted: \t{}".format(the_scenario.CO2urbanRestricted)) - logger.config("xml_CO2ruralUnrestricted: \t{}".format(the_scenario.CO2ruralUnrestricted)) - logger.config("xml_CO2ruralRestricted: \t{}".format(the_scenario.CO2ruralRestricted)) - logger.config("xml_railroadCO2Emissions: \t{}".format(round(the_scenario.railroadCO2Emissions,2))) - logger.config("xml_bargeCO2Emissions: \t{}".format(round(the_scenario.bargeCO2Emissions,2))) - logger.config("xml_pipelineCO2Emissions: \t{}".format(the_scenario.pipelineCO2Emissions)) - - logger.config("xml_ndrOn: \t{}".format(the_scenario.ndrOn)) - logger.config("xml_permittedModes: \t{}".format(the_scenario.permittedModes)) - logger.config("xml_capacityOn: \t{}".format(the_scenario.capacityOn)) - logger.config("xml_backgroundFlowModes: \t{}".format(the_scenario.backgroundFlowModes)) - logger.config("xml_minCapacityLevel: \t{}".format(the_scenario.minCapacityLevel)) - logger.config("xml_unMetDemandPenalty: \t{}".format(the_scenario.unMetDemandPenalty)) - -#======================================================================================================================= - - -def create_scenario_config_db(the_scenario, logger): - logger.debug("starting make_scenario_config_db") - - # dump the scenario into a db so that FTOT can warn user about any config changes within a scenario run - with sqlite3.connect(the_scenario.main_db) as db_con: - - # drop the table - sql = "drop table if exists scenario_config" - db_con.execute(sql) - - # create the table - sql = """create table scenario_config( - permitted_modes text, - capacity_on text, - background_flow_modes text - );""" - - db_con.execute(sql) - - config_list = [] - # format for the optimal route segments table - config_list.append([str(the_scenario.permittedModes), - the_scenario.capacityOn, - str(the_scenario.backgroundFlowModes)]) - - logger.debug("done making the config_list") - with sqlite3.connect(the_scenario.main_db) as db_con: - insert_sql = """ - INSERT into scenario_config - values (?,?,?) - ;""" - - db_con.executemany(insert_sql, config_list) - logger.debug("finish scenario_config db_con.executemany()") - db_con.commit() - logger.debug("finish scenario_config db_con.commit") - -def check_scenario_config_db(the_scenario, logger): - logger.debug("checking consistency of scenario config file with previous step") - - -def create_network_config_id_table(the_scenario, logger): - logger.info("start: create_network_config_id_table") - - # connect to the database and set the values - # ------------------------------------------ - with sqlite3.connect(the_scenario.routes_cache) as db_con: - # populate the network configuration table with the network config from the scenario object - # network_config_id INTEGER PRIMARY KEY, - - sql = """ - insert into network_config (network_config_id, - network_template, - intermodal_network, - road_artificial_link_dist, - rail_artificial_link_dist, - water_artificial_link_dist, - pipeline_crude_artificial_link_dist, - pipeline_prod_artificial_link_dist, - liquid_railroad_class_I_cost, - solid_railroad_class_I_cost, - railroad_density_code_7, - railroad_density_code_6, - railroad_density_code_5, - railroad_density_code_4, - railroad_density_code_3, - railroad_density_code_2, - railroad_density_code_1, - railroad_density_code_0, - liquid_truck_base_cost, - solid_truck_base_cost, - truck_interstate_weight, - truck_principal_arterial_weight, - truck_minor_arterial_weight, - truck_local_weight, - liquid_barge_cost, - solid_barge_cost, - water_high_vol, - water_med_vol, - water_low_vol, - water_no_vol, - solid_transloading_cost, - liquid_transloading_cost, - liquid_rail_short_haul_penalty, - solid_rail_short_haul_penalty, - liquid_water_short_haul_penalty, - solid_water_short_haul_penalty - ) - values ( - NULL, '{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});""".format( - the_scenario.template_network_gdb, - the_scenario.base_network_gdb, - the_scenario.road_max_artificial_link_dist, - the_scenario.rail_max_artificial_link_dist, - the_scenario.water_max_artificial_link_dist, - the_scenario.pipeline_crude_max_artificial_link_dist, - the_scenario.pipeline_prod_max_artificial_link_dist, - the_scenario.liquid_railroad_class_1_cost.magnitude, - the_scenario.solid_railroad_class_1_cost.magnitude, - the_scenario.rail_dc_7, - the_scenario.rail_dc_6, - the_scenario.rail_dc_5, - the_scenario.rail_dc_4, - the_scenario.rail_dc_3, - the_scenario.rail_dc_2, - the_scenario.rail_dc_1, - the_scenario.rail_dc_0, - - the_scenario.liquid_truck_base_cost.magnitude, - the_scenario.solid_truck_base_cost.magnitude, - the_scenario.truck_interstate, - the_scenario.truck_pr_art, - the_scenario.truck_m_art, - the_scenario.truck_local, - - the_scenario.liquid_barge_cost.magnitude, - the_scenario.solid_barge_cost.magnitude, - the_scenario.water_high_vol, - the_scenario.water_med_vol, - the_scenario.water_low_vol, - the_scenario.water_no_vol, - - the_scenario.solid_transloading_cost.magnitude, - the_scenario.liquid_transloading_cost.magnitude, - - the_scenario.liquid_rail_short_haul_penalty, - the_scenario.solid_rail_short_haul_penalty, - the_scenario.liquid_water_short_haul_penalty, - the_scenario.solid_water_short_haul_penalty) - - db_con.execute(sql) - - logger.debug("finish: create_network_config_id_table") - - -# ============================================================================== - -def get_network_config_id(the_scenario, logger): - logger.info("start: get_network_config_id") - network_config_id = 0 - - # check if the database exists - try: - # connect to the database and get the network_config_id that matches the scenario - # ------------------------------------------------------------------------------- - with sqlite3.connect(the_scenario.routes_cache) as db_con: - - sql = """select network_config_id - from network_config - where - network_template = '{}' and - intermodal_network = '{}' and - road_artificial_link_dist = {} and - rail_artificial_link_dist = {} and - water_artificial_link_dist = {} and - pipeline_crude_artificial_link_dist = {} and - pipeline_prod_artificial_link_dist = {} and - liquid_railroad_class_I_cost = {} and - solid_railroad_class_I_cost = {} and - railroad_density_code_7 = {} and - railroad_density_code_6 = {} and - railroad_density_code_5 = {} and - railroad_density_code_4 = {} and - railroad_density_code_3 = {} and - railroad_density_code_2 = {} and - railroad_density_code_1 = {} and - railroad_density_code_0 = {} and - liquid_truck_base_cost = {} and - solid_truck_base_cost = {} and - truck_interstate_weight = {} and - truck_principal_arterial_weight = {} and - truck_minor_arterial_weight = {} and - truck_local_weight = {} and - liquid_barge_cost = {} and - solid_barge_cost = {} and - water_high_vol = {} and - water_med_vol = {} and - water_low_vol = {} and - water_no_vol = {} and - solid_transloading_cost = {} and - liquid_transloading_cost = {} and - liquid_rail_short_haul_penalty = {} and - solid_rail_short_haul_penalty = {} and - liquid_water_short_haul_penalty = {} and - solid_water_short_haul_penalty = {} - ; """.format( - the_scenario.template_network_gdb, - the_scenario.base_network_gdb, - the_scenario.road_max_artificial_link_dist, - the_scenario.rail_max_artificial_link_dist, - the_scenario.water_max_artificial_link_dist, - the_scenario.pipeline_crude_max_artificial_link_dist, - the_scenario.pipeline_prod_max_artificial_link_dist, - - the_scenario.liquid_railroad_class_1_cost.magnitude, - the_scenario.solid_railroad_class_1_cost.magnitude, - the_scenario.rail_dc_7, - the_scenario.rail_dc_6, - the_scenario.rail_dc_5, - the_scenario.rail_dc_4, - the_scenario.rail_dc_3, - the_scenario.rail_dc_2, - the_scenario.rail_dc_1, - the_scenario.rail_dc_0, - - the_scenario.liquid_truck_base_cost.magnitude, - the_scenario.solid_truck_base_cost.magnitude, - the_scenario.truck_interstate, - the_scenario.truck_pr_art, - the_scenario.truck_m_art, - the_scenario.truck_local, - - the_scenario.liquid_barge_cost.magnitude, - the_scenario.solid_barge_cost.magnitude, - the_scenario.water_high_vol, - the_scenario.water_med_vol, - the_scenario.water_low_vol, - the_scenario.water_no_vol, - - the_scenario.solid_transloading_cost.magnitude, - the_scenario.liquid_transloading_cost.magnitude, - - the_scenario.liquid_rail_short_haul_penalty, - the_scenario.solid_rail_short_haul_penalty, - the_scenario.liquid_water_short_haul_penalty, - the_scenario.solid_water_short_haul_penalty) - - db_cur = db_con.execute(sql) - network_config_id = db_cur.fetchone()[0] - except: - warning = "could not retrieve network configuration id from the routes_cache. likely, it doesn't exist yet" - logger.debug(warning) - - # if the id is 0 it couldnt find it in the entry. now try it to the DB - if network_config_id == 0: - create_network_config_id_table(the_scenario, logger) - - return network_config_id + +#--------------------------------------------------------------------------------------------------- +# Name: ftot_scenario +# +# Purpose: declare all of the attributes of the_scenario object. +# create getter and setter methods for each attribute. +# +#--------------------------------------------------------------------------------------------------- + + + +import os +import sys +from xml.dom import minidom +from ftot import SCHEMA_VERSION +from ftot import Q_, ureg +import sqlite3 + +try: + from lxml import etree +except ImportError: + print ("This script requires the lxml Python library to validate the XML scenario file.") + print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") + print("Exiting...") + sys.exit() + +#=================================================================================================== + +def getElementFromXmlFile(xmlFile, elementName): + return xmlFile.getElementsByTagName(elementName)[0].firstChild.data + +#=================================================================================================== + +def format_number(numString): + """Removes any number formatting, i.e., thousand's separator or dollar sign""" + + if numString.rfind(",") > -1: + numString = numString.replace(",", "") + if numString.rfind("$") > -1: + numString = numString.replace("$", "") + return float(numString) + +#=================================================================================================== + +class Scenario: + def __init__(self): + pass + +def load_scenario_config_file(fullPathToXmlConfigFile, fullPathToXmlSchemaFile, logger): + + if not os.path.exists(fullPathToXmlConfigFile): + raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) + + if fullPathToXmlConfigFile.rfind(".xml") < 0: + raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) + + if not os.path.exists(fullPathToXmlSchemaFile): + raise IOError("XML Schema File not found at {}".format(fullPathToXmlSchemaFile)) + + xmlScenarioFile = minidom.parse(fullPathToXmlConfigFile) + + # Validate XML scenario against XML schema + schemaObj = etree.XMLSchema(etree.parse(fullPathToXmlSchemaFile)) + xmlFileObj = etree.parse(fullPathToXmlConfigFile) + validationResult = schemaObj.validate(xmlFileObj) + + logger.debug("validate XML scenario against XML schema") + if validationResult == False: + logger.warning("XML scenario validation failed. Error messages to follow.") + for error in schemaObj.error_log: + logger.warning("ERROR ON LINE: {} - ERROR MESSAGE: {}".format(error.line, error.message)) + + raise Exception("XML Scenario File does not meet the requirements in the XML schema file.") + + # initialize scenario ojbect + logger.debug("initialize scenario object") + scenario = Scenario() + + # Check the scenario schema version (current version is specified in FTOT.py under VERSION_NUMBER global var) + logger.debug("validate schema version is correct") + scenario.scenario_schema_version = xmlScenarioFile.getElementsByTagName('Scenario_Schema_Version')[0].firstChild.data + + #if not str(VERSION_NUMBER) == str(scenario.scenario_schema_version): + if not str(SCHEMA_VERSION).split(".")[0:2] == str(scenario.scenario_schema_version).split(".")[0:2]: + error = "XML Schema File Version is {}. Expected version {}. " \ + "Use the XML flag to run the XML upgrade tool. " \ + .format(str(scenario.scenario_schema_version), str(SCHEMA_VERSION)) + logger.error(error) + raise Exception(error) + + scenario.scenario_name = xmlScenarioFile.getElementsByTagName('Scenario_Name')[0].firstChild.data + # Convert any commas in the scenario name to dashes + warning = "Replace any commas in the scenario name with dashes to accomodate csv files." + logger.debug(warning) + scenario.scenario_name = scenario.scenario_name.replace(",", "-") + scenario.scenario_description = xmlScenarioFile.getElementsByTagName('Scenario_Description')[0].firstChild.data + + # SCENARIO INPUTS SECTION + # ---------------------------------------------------------------------------------------- + scenario.common_data_folder = xmlScenarioFile.getElementsByTagName('Common_Data_Folder')[0].firstChild.data + scenario.base_network_gdb = xmlScenarioFile.getElementsByTagName('Base_Network_Gdb')[0].firstChild.data + scenario.disruption_data = xmlScenarioFile.getElementsByTagName('Disruption_Data')[0].firstChild.data + scenario.base_rmp_layer = xmlScenarioFile.getElementsByTagName('Base_RMP_Layer')[0].firstChild.data + scenario.base_destination_layer = xmlScenarioFile.getElementsByTagName('Base_Destination_Layer')[0].firstChild.data + scenario.base_processors_layer = xmlScenarioFile.getElementsByTagName('Base_Processors_Layer')[0].firstChild.data + + scenario.rmp_commodity_data = xmlScenarioFile.getElementsByTagName('RMP_Commodity_Data')[0].firstChild.data + scenario.destinations_commodity_data = xmlScenarioFile.getElementsByTagName('Destinations_Commodity_Data')[0].firstChild.data + scenario.processors_commodity_data = xmlScenarioFile.getElementsByTagName('Processors_Commodity_Data')[0].firstChild.data + scenario.processors_candidate_slate_data = xmlScenarioFile.getElementsByTagName('Processors_Candidate_Commodity_Data')[0].firstChild.data + # note: the processor_candidates_data is defined under other since it is not a user specified file. + scenario.schedule = xmlScenarioFile.getElementsByTagName('Schedule_Data')[0].firstChild.data + scenario.commodity_mode_data = xmlScenarioFile.getElementsByTagName('Commodity_Mode_Data')[0].firstChild.data + + # use pint to set the default units + logger.debug("test: setting the default units with pint") + try: + scenario.default_units_solid_phase = Q_(xmlScenarioFile.getElementsByTagName('Default_Units_Solid_Phase')[0].firstChild.data).units + scenario.default_units_liquid_phase = Q_(xmlScenarioFile.getElementsByTagName('Default_Units_Liquid_Phase')[0].firstChild.data).units + logger.debug("PASS: setting the default units with pint") + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # ASSUMPTIONS SECTION + # ---------------------------------------------------------------------------------------- + + # solid and liquid vehicle loads + try: + + logger.debug("test: setting the vehicle loads for solid phase of matter with pint") + scenario.truck_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Truck_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) + scenario.railcar_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Railcar_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) + scenario.barge_load_solid = Q_(xmlScenarioFile.getElementsByTagName('Barge_Load_Solid')[0].firstChild.data).to(scenario.default_units_solid_phase) + + logger.debug("test: setting the vehicle loads for liquid phase of matter with pint") + scenario.truck_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Truck_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) + scenario.railcar_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Railcar_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) + scenario.barge_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Barge_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) + scenario.pipeline_crude_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Crude_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) + scenario.pipeline_prod_load_liquid = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Prod_Load_Liquid')[0].firstChild.data).to(scenario.default_units_liquid_phase) + logger.debug("PASS: setting the vehicle loads with pint passed") + + logger.debug("test: setting the vehicle fuel efficiencies with pint") + scenario.truckFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Truck_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') + scenario.railFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Rail_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') + scenario.bargeFuelEfficiency = Q_(xmlScenarioFile.getElementsByTagName('Barge_Fuel_Efficiency')[0].firstChild.data).to('mi/gal') + logger.debug("PASS: setting the vehicle fuel efficiencies with pint passed") + + logger.debug("test: setting the vehicle emission factors with pint") + scenario.CO2urbanUnrestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Urban_Unrestricted')[0].firstChild.data).to('g/mi') + scenario.CO2urbanRestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Urban_Restricted')[0].firstChild.data).to('g/mi') + scenario.CO2ruralUnrestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Rural_Unrestricted')[0].firstChild.data).to('g/mi') + scenario.CO2ruralRestricted = Q_(xmlScenarioFile.getElementsByTagName('Atmos_CO2_Rural_Restricted')[0].firstChild.data).to('g/mi') + scenario.railroadCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Railroad_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) + scenario.bargeCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Barge_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) + scenario.pipelineCO2Emissions = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_CO2_Emissions')[0].firstChild.data).to('g/{}/mi'.format(scenario.default_units_solid_phase)) + # setting density conversion based on 'Density_Conversion_Factor' field if it exists, or otherwise default to 3.33 ton/kgal + if len(xmlScenarioFile.getElementsByTagName('Density_Conversion_Factor')): + scenario.densityFactor = Q_(xmlScenarioFile.getElementsByTagName('Density_Conversion_Factor')[0].firstChild.data).to('{}/{}'.format(scenario.default_units_solid_phase, scenario.default_units_liquid_phase)) + else: + logger.warning("FTOT is assuming a density of 3.33 ton/kgal for emissions reporting for liquids. Use scenario XML parameter 'Density_Conversion_Factor' to adjust this value.") + scenario.densityFactor = Q_('3.33 ton/kgal').to('{}/{}'.format(scenario.default_units_solid_phase, scenario.default_units_liquid_phase)) + logger.debug("PASS: setting the vehicle emission factors with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # Setting flag for detailed emissions reporting if it exists, or otherwise default to 'False' + if len(xmlScenarioFile.getElementsByTagName('Detailed_Emissions_Reporting')): + if xmlScenarioFile.getElementsByTagName('Detailed_Emissions_Reporting')[0].firstChild.data == "True": + scenario.detailed_emissions = True + else: + scenario.detailed_emissions = False + else: + logger.debug("Detailed_Emissions_Reporting field not specified. Defaulting to False.") + scenario.detailed_emissions = False + + # SCRIPT PARAMETERS SECTION FOR NETWORK + # ---------------------------------------------------------------------------------------- + + # rail costs + try: + logger.debug("test: setting the base costs for rail with pint") + scenario.solid_railroad_class_1_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Railroad_Class_I_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) + scenario.liquid_railroad_class_1_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Railroad_Class_I_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) + logger.debug("PASS: setting the base costs for rail with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # rail penalties + scenario.rail_dc_7 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_7_Weight')[0].firstChild.data) + scenario.rail_dc_6 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_6_Weight')[0].firstChild.data) + scenario.rail_dc_5 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_5_Weight')[0].firstChild.data) + scenario.rail_dc_4 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_4_Weight')[0].firstChild.data) + scenario.rail_dc_3 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_3_Weight')[0].firstChild.data) + scenario.rail_dc_2 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_2_Weight')[0].firstChild.data) + scenario.rail_dc_1 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_1_Weight')[0].firstChild.data) + scenario.rail_dc_0 = format_number(xmlScenarioFile.getElementsByTagName('Rail_Density_Code_0_Weight')[0].firstChild.data) + + # truck costs + try: + logger.debug("test: setting the base costs for truck with pint") + scenario.solid_truck_base_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Truck_Base_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) + scenario.liquid_truck_base_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Truck_Base_Cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) + logger.debug("PASS: setting the base costs for truck with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # road penalties + scenario.truck_interstate = format_number(xmlScenarioFile.getElementsByTagName('Truck_Interstate_Weight')[0].firstChild.data) + scenario.truck_pr_art = format_number(xmlScenarioFile.getElementsByTagName('Truck_Principal_Arterial_Weight')[0].firstChild.data) + scenario.truck_m_art = format_number(xmlScenarioFile.getElementsByTagName('Truck_Minor_Arterial_Weight')[0].firstChild.data) + scenario.truck_local = format_number(xmlScenarioFile.getElementsByTagName('Truck_Local_Weight')[0].firstChild.data) + + # barge costs + try: + logger.debug("test: setting the base costs for barge with pint") + scenario.solid_barge_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Barge_cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_solid_phase)) + scenario.liquid_barge_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Barge_cost')[0].firstChild.data).to("usd/{}/mile".format(scenario.default_units_liquid_phase)) + logger.debug("PASS: setting the base costs for barge with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # water penalties + scenario.water_high_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_High_Volume_Weight')[0].firstChild.data) + scenario.water_med_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_Medium_Volume_Weight')[0].firstChild.data) + scenario.water_low_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_Low_Volume_Weight')[0].firstChild.data) + scenario.water_no_vol = format_number(xmlScenarioFile.getElementsByTagName('Water_No_Volume_Weight')[0].firstChild.data) + + # transloading costs + try: + logger.debug("test: setting the transloading costs with pint") + scenario.solid_transloading_cost = Q_(xmlScenarioFile.getElementsByTagName('solid_Transloading_Cost')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) + scenario.liquid_transloading_cost = Q_(xmlScenarioFile.getElementsByTagName('liquid_Transloading_Cost')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) + logger.debug("PASS: setting the transloading costs with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # artificial link distances + try: + logger.debug("test: setting the artificial link distances with pint") + scenario.road_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Road_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') + scenario.rail_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Rail_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') + scenario.water_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Water_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') + scenario.pipeline_crude_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Crude_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') + scenario.pipeline_prod_max_artificial_link_dist = Q_(xmlScenarioFile.getElementsByTagName('Pipeline_Products_Max_Artificial_Link_Distance')[0].firstChild.data).to('mi') + logger.debug("PASS: setting the artificial link distances with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # short haul penalties + try: + logger.debug("test: setting the short haul penalties with pint") + scenario.liquid_rail_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('liquid_Rail_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) + scenario.solid_rail_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('solid_Rail_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) + scenario.liquid_water_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('liquid_Water_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_liquid_phase)) + scenario.solid_water_short_haul_penalty = Q_(xmlScenarioFile.getElementsByTagName('solid_Water_Short_Haul_Penalty')[0].firstChild.data).to("usd/{}".format(scenario.default_units_solid_phase)) + logger.debug("PASS: setting the short haul penalties with pint passed") + + except Exception as e: + logger.error("FAIL: {} ".format(e)) + raise Exception("FAIL: {}".format(e)) + + # RUN ROUTE OPTIMIZATION SCRIPT SECTION + # ---------------------------------------------------------------------------------------- + + # Setting flag for network density reduction based on 'NDR_On' field + if xmlScenarioFile.getElementsByTagName('NDR_On')[0].firstChild.data == "True": + scenario.ndrOn = True + else: + scenario.ndrOn = False + + scenario.permittedModes = [] + if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Road')[0].firstChild.data == "True": + scenario.permittedModes.append("road") + if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Rail')[0].firstChild.data == "True": + scenario.permittedModes.append("rail") + if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Water')[0].firstChild.data == "True": + scenario.permittedModes.append("water") + + # TODO ALO-- 10/17/2018-- make below compatible with distinct crude/product pipeline approach-- ftot_pulp.py changes will be needed. + if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Pipeline_Crude')[0].firstChild.data == "True": + scenario.permittedModes.append("pipeline_crude_trf_rts") + if xmlScenarioFile.getElementsByTagName('Permitted_Modes')[0].getElementsByTagName('Pipeline_Prod')[0].firstChild.data == "True": + scenario.permittedModes.append("pipeline_prod_trf_rts") + + if xmlScenarioFile.getElementsByTagName('Capacity_On')[0].firstChild.data == "True": + scenario.capacityOn = True + else: + scenario.capacityOn = False + + scenario.backgroundFlowModes = [] + if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Road')[0].firstChild.data == "True": + scenario.backgroundFlowModes.append("road") + if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Rail')[0].firstChild.data == "True": + scenario.backgroundFlowModes.append("rail") + if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Water')[0].firstChild.data == "True": + scenario.backgroundFlowModes.append("water") + + if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Pipeline_Crude')[0].firstChild.data == "True": + scenario.backgroundFlowModes.append("pipeline") + if xmlScenarioFile.getElementsByTagName('Background_Flows')[0].getElementsByTagName('Pipeline_Prod')[0].firstChild.data == "True": + scenario.backgroundFlowModes.append("pipeline") + + scenario.minCapacityLevel = float(xmlScenarioFile.getElementsByTagName('Minimum_Capacity_Level')[0].firstChild.data) + + scenario.unMetDemandPenalty = float(xmlScenarioFile.getElementsByTagName('Unmet_Demand_Penalty')[0].firstChild.data) + + # OTHER + # ---------------------------------------------------------------------------------------- + + scenario.scenario_run_directory = os.path.dirname(fullPathToXmlConfigFile) + + scenario.main_db = os.path.join(scenario.scenario_run_directory, "main.db") + scenario.main_gdb = os.path.join(scenario.scenario_run_directory, "main.gdb") + + scenario.rmp_fc = os.path.join(scenario.main_gdb, "raw_material_producers") + scenario.destinations_fc = os.path.join(scenario.main_gdb, "ultimate_destinations") + scenario.processors_fc = os.path.join(scenario.main_gdb, "processors") + scenario.processor_candidates_fc = os.path.join(scenario.main_gdb, "all_candidate_processors") + scenario.locations_fc = os.path.join(scenario.main_gdb, "locations") + + # this file is generated by the processor_candidates() method + scenario.processor_candidates_commodity_data = os.path.join(scenario.scenario_run_directory, "debug", "ftot_generated_processor_candidates.csv") + + # this is the directory to store the shp files that a programtically generated for the networkx read_shp method + scenario.networkx_files_dir = os.path.join(scenario.scenario_run_directory, "temp_networkx_shp_files") + + return scenario + +#=================================================================================================== + + +def dump_scenario_info_to_report(the_scenario, logger): + logger.config("xml_scenario_name: \t{}".format(the_scenario.scenario_name)) + logger.config("xml_scenario_description: \t{}".format(the_scenario.scenario_description)) + logger.config("xml_scenario_run_directory: \t{}".format(the_scenario.scenario_run_directory)) + logger.config("xml_scenario_schema_version: \t{}".format(the_scenario.scenario_schema_version)) + + logger.config("xml_common_data_folder: \t{}".format(the_scenario.common_data_folder)) + + logger.config("xml_base_network_gdb: \t{}".format(the_scenario.base_network_gdb)) + logger.config("xml_disruption_data: \t{}".format(the_scenario.disruption_data)) + + logger.config("xml_base_rmp_layer: \t{}".format(the_scenario.base_rmp_layer)) + logger.config("xml_base_destination_layer: \t{}".format(the_scenario.base_destination_layer)) + logger.config("xml_base_processors_layer: \t{}".format(the_scenario.base_processors_layer)) + + logger.config("xml_rmp_commodity_data: \t{}".format(the_scenario.rmp_commodity_data)) + logger.config("xml_destinations_commodity_data: \t{}".format(the_scenario.destinations_commodity_data)) + logger.config("xml_processors_commodity_data: \t{}".format(the_scenario.processors_commodity_data)) + logger.config("xml_processors_candidate_slate_data: \t{}".format(the_scenario.processors_candidate_slate_data)) + + logger.config("xml_schedule_data: \t{}".format(the_scenario.schedule)) + logger.config("xml_commodity_mode_data: \t{}".format(the_scenario.commodity_mode_data)) + + logger.config("xml_default_units_solid_phase: \t{}".format(the_scenario.default_units_solid_phase)) + logger.config("xml_default_units_liquid_phase: \t{}".format(the_scenario.default_units_liquid_phase)) + + logger.config("xml_truck_load_solid: \t{}".format(the_scenario.truck_load_solid)) + logger.config("xml_railcar_load_solid: \t{}".format(the_scenario.railcar_load_solid)) + logger.config("xml_barge_load_solid: \t{}".format(the_scenario.barge_load_solid)) + + logger.config("xml_truck_load_liquid: \t{}".format(the_scenario.truck_load_liquid)) + logger.config("xml_railcar_load_liquid: \t{}".format(the_scenario.railcar_load_liquid)) + logger.config("xml_barge_load_liquid: \t{}".format(the_scenario.barge_load_liquid)) + logger.config("xml_pipeline_crude_load_liquid: \t{}".format(the_scenario.pipeline_crude_load_liquid)) + logger.config("xml_pipeline_prod_load_liquid: \t{}".format(the_scenario.pipeline_prod_load_liquid)) + + logger.config("xml_solid_railroad_class_1_cost: \t{}".format(the_scenario.solid_railroad_class_1_cost)) + logger.config("xml_liquid_railroad_class_1_cost: \t{}".format(the_scenario.liquid_railroad_class_1_cost)) + logger.config("xml_rail_dc_7: \t{}".format(the_scenario.rail_dc_7)) + logger.config("xml_rail_dc_6: \t{}".format(the_scenario.rail_dc_6)) + logger.config("xml_rail_dc_5: \t{}".format(the_scenario.rail_dc_5)) + logger.config("xml_rail_dc_4: \t{}".format(the_scenario.rail_dc_4)) + logger.config("xml_rail_dc_3: \t{}".format(the_scenario.rail_dc_3)) + logger.config("xml_rail_dc_2: \t{}".format(the_scenario.rail_dc_2)) + logger.config("xml_rail_dc_1: \t{}".format(the_scenario.rail_dc_1)) + logger.config("xml_rail_dc_0: \t{}".format(the_scenario.rail_dc_0)) + + logger.config("xml_liquid_truck_base_cost: \t{}".format(the_scenario.liquid_truck_base_cost)) + logger.config("xml_solid_truck_base_cost: \t{}".format(the_scenario.solid_truck_base_cost)) + logger.config("xml_truck_interstate: \t{}".format(the_scenario.truck_interstate)) + logger.config("xml_truck_pr_art: \t{}".format(the_scenario.truck_pr_art)) + logger.config("xml_truck_m_art: \t{}".format(the_scenario.truck_m_art)) + logger.config("xml_truck_local: \t{}".format(the_scenario.truck_local)) + + logger.config("xml_liquid_barge_cost: \t{}".format(the_scenario.liquid_barge_cost)) + logger.config("xml_solid_barge_cost: \t{}".format(the_scenario.solid_barge_cost)) + logger.config("xml_water_high_vol: \t{}".format(the_scenario.water_high_vol)) + logger.config("xml_water_med_vol: \t{}".format(the_scenario.water_med_vol)) + logger.config("xml_water_low_vol: \t{}".format(the_scenario.water_low_vol)) + logger.config("xml_water_no_vol: \t{}".format(the_scenario.water_no_vol)) + + logger.config("xml_solid_transloading_cost: \t{}".format(the_scenario.solid_transloading_cost)) + logger.config("xml_liquid_transloading_cost: \t{}".format(the_scenario.liquid_transloading_cost)) + + logger.config("xml_road_max_artificial_link_dist: \t{}".format(the_scenario.road_max_artificial_link_dist)) + logger.config("xml_rail_max_artificial_link_dist: \t{}".format(the_scenario.rail_max_artificial_link_dist)) + logger.config("xml_water_max_artificial_link_dist: \t{}".format(the_scenario.water_max_artificial_link_dist)) + logger.config("xml_pipeline_crude_max_artificial_link_dist: \t{}".format(the_scenario.pipeline_crude_max_artificial_link_dist)) + logger.config("xml_pipeline_prod_max_artificial_link_dist: \t{}".format(the_scenario.pipeline_prod_max_artificial_link_dist)) + + logger.config("xml_liquid_rail_short_haul_penalty: \t{}".format(the_scenario.liquid_rail_short_haul_penalty)) + logger.config("xml_solid_rail_short_haul_penalty: \t{}".format(the_scenario.solid_rail_short_haul_penalty)) + logger.config("xml_liquid_water_short_haul_penalty: \t{}".format(the_scenario.liquid_water_short_haul_penalty)) + logger.config("xml_solid_water_short_haul_penalty: \t{}".format(the_scenario.solid_water_short_haul_penalty)) + + logger.config("xml_truckFuelEfficiency: \t{}".format(the_scenario.truckFuelEfficiency)) + logger.config("xml_bargeFuelEfficiency: \t{}".format(the_scenario.bargeFuelEfficiency)) + logger.config("xml_railFuelEfficiency: \t{}".format(the_scenario.railFuelEfficiency)) + logger.config("xml_CO2urbanUnrestricted: \t{}".format(the_scenario.CO2urbanUnrestricted)) + logger.config("xml_CO2urbanRestricted: \t{}".format(the_scenario.CO2urbanRestricted)) + logger.config("xml_CO2ruralUnrestricted: \t{}".format(the_scenario.CO2ruralUnrestricted)) + logger.config("xml_CO2ruralRestricted: \t{}".format(the_scenario.CO2ruralRestricted)) + logger.config("xml_railroadCO2Emissions: \t{}".format(round(the_scenario.railroadCO2Emissions,2))) + logger.config("xml_bargeCO2Emissions: \t{}".format(round(the_scenario.bargeCO2Emissions,2))) + logger.config("xml_pipelineCO2Emissions: \t{}".format(the_scenario.pipelineCO2Emissions)) + logger.config("xml_densityFactor: \t{}".format(the_scenario.densityFactor)) + + logger.config("xml_detailed_emissions: \t{}".format(the_scenario.detailed_emissions)) + logger.config("xml_ndrOn: \t{}".format(the_scenario.ndrOn)) + logger.config("xml_permittedModes: \t{}".format(the_scenario.permittedModes)) + logger.config("xml_capacityOn: \t{}".format(the_scenario.capacityOn)) + logger.config("xml_backgroundFlowModes: \t{}".format(the_scenario.backgroundFlowModes)) + logger.config("xml_minCapacityLevel: \t{}".format(the_scenario.minCapacityLevel)) + logger.config("xml_unMetDemandPenalty: \t{}".format(the_scenario.unMetDemandPenalty)) + +#======================================================================================================================= + + +def create_scenario_config_db(the_scenario, logger): + logger.debug("starting make_scenario_config_db") + + # dump the scenario into a db so that FTOT can warn user about any config changes within a scenario run + with sqlite3.connect(the_scenario.main_db) as db_con: + + # drop the table + sql = "drop table if exists scenario_config" + db_con.execute(sql) + + # create the table + sql = """create table scenario_config( + permitted_modes text, + capacity_on text, + background_flow_modes text + );""" + + db_con.execute(sql) + + config_list = [] + # format for the optimal route segments table + config_list.append([str(the_scenario.permittedModes), + the_scenario.capacityOn, + str(the_scenario.backgroundFlowModes)]) + + logger.debug("done making the config_list") + with sqlite3.connect(the_scenario.main_db) as db_con: + insert_sql = """ + INSERT into scenario_config + values (?,?,?) + ;""" + + db_con.executemany(insert_sql, config_list) + logger.debug("finish scenario_config db_con.executemany()") + db_con.commit() + logger.debug("finish scenario_config db_con.commit") + +def check_scenario_config_db(the_scenario, logger): + logger.debug("checking consistency of scenario config file with previous step") + + +def create_network_config_id_table(the_scenario, logger): + logger.info("start: create_network_config_id_table") + + # connect to the database and set the values + # ------------------------------------------ + with sqlite3.connect(the_scenario.routes_cache) as db_con: + # populate the network configuration table with the network config from the scenario object + # network_config_id INTEGER PRIMARY KEY, + + sql = """ + insert into network_config (network_config_id, + network_template, + intermodal_network, + road_artificial_link_dist, + rail_artificial_link_dist, + water_artificial_link_dist, + pipeline_crude_artificial_link_dist, + pipeline_prod_artificial_link_dist, + liquid_railroad_class_I_cost, + solid_railroad_class_I_cost, + railroad_density_code_7, + railroad_density_code_6, + railroad_density_code_5, + railroad_density_code_4, + railroad_density_code_3, + railroad_density_code_2, + railroad_density_code_1, + railroad_density_code_0, + liquid_truck_base_cost, + solid_truck_base_cost, + truck_interstate_weight, + truck_principal_arterial_weight, + truck_minor_arterial_weight, + truck_local_weight, + liquid_barge_cost, + solid_barge_cost, + water_high_vol, + water_med_vol, + water_low_vol, + water_no_vol, + solid_transloading_cost, + liquid_transloading_cost, + liquid_rail_short_haul_penalty, + solid_rail_short_haul_penalty, + liquid_water_short_haul_penalty, + solid_water_short_haul_penalty + ) + values ( + NULL, '{}', '{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {});""".format( + the_scenario.template_network_gdb, + the_scenario.base_network_gdb, + the_scenario.road_max_artificial_link_dist, + the_scenario.rail_max_artificial_link_dist, + the_scenario.water_max_artificial_link_dist, + the_scenario.pipeline_crude_max_artificial_link_dist, + the_scenario.pipeline_prod_max_artificial_link_dist, + the_scenario.liquid_railroad_class_1_cost.magnitude, + the_scenario.solid_railroad_class_1_cost.magnitude, + the_scenario.rail_dc_7, + the_scenario.rail_dc_6, + the_scenario.rail_dc_5, + the_scenario.rail_dc_4, + the_scenario.rail_dc_3, + the_scenario.rail_dc_2, + the_scenario.rail_dc_1, + the_scenario.rail_dc_0, + + the_scenario.liquid_truck_base_cost.magnitude, + the_scenario.solid_truck_base_cost.magnitude, + the_scenario.truck_interstate, + the_scenario.truck_pr_art, + the_scenario.truck_m_art, + the_scenario.truck_local, + + the_scenario.liquid_barge_cost.magnitude, + the_scenario.solid_barge_cost.magnitude, + the_scenario.water_high_vol, + the_scenario.water_med_vol, + the_scenario.water_low_vol, + the_scenario.water_no_vol, + + the_scenario.solid_transloading_cost.magnitude, + the_scenario.liquid_transloading_cost.magnitude, + + the_scenario.liquid_rail_short_haul_penalty, + the_scenario.solid_rail_short_haul_penalty, + the_scenario.liquid_water_short_haul_penalty, + the_scenario.solid_water_short_haul_penalty) + + db_con.execute(sql) + + logger.debug("finish: create_network_config_id_table") + + +# ============================================================================== + +def get_network_config_id(the_scenario, logger): + logger.info("start: get_network_config_id") + network_config_id = 0 + + # check if the database exists + try: + # connect to the database and get the network_config_id that matches the scenario + # ------------------------------------------------------------------------------- + with sqlite3.connect(the_scenario.routes_cache) as db_con: + + sql = """select network_config_id + from network_config + where + network_template = '{}' and + intermodal_network = '{}' and + road_artificial_link_dist = {} and + rail_artificial_link_dist = {} and + water_artificial_link_dist = {} and + pipeline_crude_artificial_link_dist = {} and + pipeline_prod_artificial_link_dist = {} and + liquid_railroad_class_I_cost = {} and + solid_railroad_class_I_cost = {} and + railroad_density_code_7 = {} and + railroad_density_code_6 = {} and + railroad_density_code_5 = {} and + railroad_density_code_4 = {} and + railroad_density_code_3 = {} and + railroad_density_code_2 = {} and + railroad_density_code_1 = {} and + railroad_density_code_0 = {} and + liquid_truck_base_cost = {} and + solid_truck_base_cost = {} and + truck_interstate_weight = {} and + truck_principal_arterial_weight = {} and + truck_minor_arterial_weight = {} and + truck_local_weight = {} and + liquid_barge_cost = {} and + solid_barge_cost = {} and + water_high_vol = {} and + water_med_vol = {} and + water_low_vol = {} and + water_no_vol = {} and + solid_transloading_cost = {} and + liquid_transloading_cost = {} and + liquid_rail_short_haul_penalty = {} and + solid_rail_short_haul_penalty = {} and + liquid_water_short_haul_penalty = {} and + solid_water_short_haul_penalty = {} + ; """.format( + the_scenario.template_network_gdb, + the_scenario.base_network_gdb, + the_scenario.road_max_artificial_link_dist, + the_scenario.rail_max_artificial_link_dist, + the_scenario.water_max_artificial_link_dist, + the_scenario.pipeline_crude_max_artificial_link_dist, + the_scenario.pipeline_prod_max_artificial_link_dist, + + the_scenario.liquid_railroad_class_1_cost.magnitude, + the_scenario.solid_railroad_class_1_cost.magnitude, + the_scenario.rail_dc_7, + the_scenario.rail_dc_6, + the_scenario.rail_dc_5, + the_scenario.rail_dc_4, + the_scenario.rail_dc_3, + the_scenario.rail_dc_2, + the_scenario.rail_dc_1, + the_scenario.rail_dc_0, + + the_scenario.liquid_truck_base_cost.magnitude, + the_scenario.solid_truck_base_cost.magnitude, + the_scenario.truck_interstate, + the_scenario.truck_pr_art, + the_scenario.truck_m_art, + the_scenario.truck_local, + + the_scenario.liquid_barge_cost.magnitude, + the_scenario.solid_barge_cost.magnitude, + the_scenario.water_high_vol, + the_scenario.water_med_vol, + the_scenario.water_low_vol, + the_scenario.water_no_vol, + + the_scenario.solid_transloading_cost.magnitude, + the_scenario.liquid_transloading_cost.magnitude, + + the_scenario.liquid_rail_short_haul_penalty, + the_scenario.solid_rail_short_haul_penalty, + the_scenario.liquid_water_short_haul_penalty, + the_scenario.solid_water_short_haul_penalty) + + db_cur = db_con.execute(sql) + network_config_id = db_cur.fetchone()[0] + except: + warning = "could not retrieve network configuration id from the routes_cache. likely, it doesn't exist yet" + logger.debug(warning) + + # if the id is 0 it couldnt find it in the entry. now try it to the DB + if network_config_id == 0: + create_network_config_id_table(the_scenario, logger) + + return network_config_id diff --git a/program/ftot_setup.py b/program/ftot_setup.py index 237a953..b383b50 100644 --- a/program/ftot_setup.py +++ b/program/ftot_setup.py @@ -1,239 +1,239 @@ - -# --------------------------------------------------------------------------------------------------- -# Name: ftot_setup -# -# Purpose: The ftot_setup module creates the main.db, and main.gdb for the scenario. The Alternative -# Fuel Production Assessment Tool (AFPAT) is also copied locally and stored in the main.gdb. -# -# --------------------------------------------------------------------------------------------------- - -import os -import datetime -import sqlite3 -from shutil import rmtree - -import ftot_supporting -import ftot_supporting_gis - -# =================================================================================================== - - -def import_afpat_data(afpat_spreadsheet, output_gdb, logger): - - import arcpy - - """ - import afpat from spreadsheet to arcgis table - """ - - logger.debug("start: import_afpat_data") - - if not os.path.exists(afpat_spreadsheet): - error = "can't find afpat spreadsheet {}".format(afpat_spreadsheet) - logger.error(error) - raise IOError(error) - - - if not os.path.exists(output_gdb): - - error = "can't find scratch gdb {}".format(output_gdb) - logger.error(error) - raise IOError(error) - - sheet = "2. User input & Results" - - out_table = os.path.join(output_gdb, "afpat_raw") - - if arcpy.Exists(out_table): - arcpy.Delete_management(out_table) - - arcpy.ExcelToTable_conversion(afpat_spreadsheet, out_table, sheet) - - logger.debug("finish: import_afpat_data") - - -# =================================================================================================== - -def cleanup(the_scenario, logger): - logger.info("start: cleanup") - - all_files = os.listdir(the_scenario.scenario_run_directory) - if all_files: - logger.info("deleting everything but the scenario .xml file and the .bat file.") - for file_or_dir in all_files: - - if file_or_dir.find("input_data") == -1: - if file_or_dir.find(".bat") == -1: - if file_or_dir.find(".xml") == -1: - try: - rmtree(os.path.join(the_scenario.scenario_run_directory, file_or_dir)) - except: - try: - os.remove(os.path.join(the_scenario.scenario_run_directory, file_or_dir)) - except: - pass - continue - all_files = os.listdir(the_scenario.scenario_run_directory) - if len(all_files) > 2: - logger.warning("something went wrong in checks_and_cleanup. There are {} extra folders and files".format(len(all_files)-2)) - else: - logger.debug("only two files left in the scenario run directory... continue") - -# =================================================================================================== - - -def setup(the_scenario, logger): - - logger.debug("start: setup") - start_time = datetime.datetime.now() - logger.info("Scenario Name: \t{}".format(the_scenario.scenario_name)) - logger.debug("Scenario Description: \t{}".format(the_scenario.scenario_description)) - logger.info("Scenario Start Date/Time: \t{}".format(start_time)) - - # create a folder for debug and intermediate files - # delete everything in there if it exists - # ------------------------------------------------ - debug_directory = os.path.join(the_scenario.scenario_run_directory, "debug") - - if os.path.exists(debug_directory): - logger.debug("deleting debug_directory and contents.") - rmtree(debug_directory) - - if not os.path.exists(debug_directory): - os.makedirs(debug_directory) - logger.debug("creating debug_directory.") - - # create the scenario database main.db - create_main_db(logger, the_scenario) - - # create the scenario geodatabase; main.gdb - create_main_gdb(logger, the_scenario) - - logger.debug("finish: SETUP: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) - - -# =================================================================================================== - - -def create_main_gdb(logger, the_scenario): - # create the GIS geodatabase main.gdb - # ----------------------------------- - logger.info("start: create_main_gdb") - scenario_gdb = the_scenario.main_gdb - - import arcpy - - if arcpy.Exists(scenario_gdb): - - logger.debug("start: delete existing main.gdb") - logger.debug("main.gdb file location: " + scenario_gdb) - try: - arcpy.Delete_management(scenario_gdb) - - except: - logger.error("Couldn't delete " + scenario_gdb) - raise Exception("Couldn't delete " + scenario_gdb) - - # copy in the base network from the baseline data - # ----------------------------------------------- - - if not arcpy.Exists(the_scenario.base_network_gdb): - error = "can't find base network gdb {}".format(the_scenario.base_network_gdb) - raise IOError(error) - - logger.info("start: copy base network to main.gdb") - logger.config("Copy Scenario Base Network: \t{}".format(the_scenario.base_network_gdb)) - arcpy.Copy_management(the_scenario.base_network_gdb, scenario_gdb) - - # Check for disruption csv-- if one exists, this is where we remove links in the network that are fully disrupted. - if the_scenario.disruption_data == "None": - logger.info('disruption file not specified. No disruption to the network will be applied.') - else: - if not os.path.exists(the_scenario.disruption_data): - logger.warning("warning: cannot find disruption_data file: {}. No disruption to the network will be applied" - .format(the_scenario.disruption_data)) - else: - # check that is actually a csv - if not the_scenario.disruption_data.endswith("csv"): - error = "error: disruption_data file: {} is not a csv file. Please use valid disruption_data csv"\ - .format(the_scenario.disruption_data) - logger.error(error) - raise Exception(error) - else: - logger.info("Disruption csv to be applied to the network: {}".format(the_scenario.disruption_data)) - with open(the_scenario.disruption_data, 'r') as rf: - line_num = 1 - for line in rf: - csv_row = line.rstrip('\n').split(',') - if line_num == 1: - if csv_row[0] != 'mode' or csv_row[1]!= 'unique_link_id' or csv_row[2] != 'link_availability': - error = "Error: disruption_data file: {} does not match the appropriate disruption "\ - "data schema. Please check that the first three columns are 'mode', "\ - "'unique_link_id' and 'link_availability'".format(the_scenario.disruption_data) - logger.error(error) - raise Exception(error) - if line_num > 1: - mode = csv_row[0] - link = csv_row[1] - link_availability = float(csv_row[2]) - if link_availability == 0: # 0 = 100% disruption - with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, "network/" + mode), ['OBJECTID'], "OBJECTID = {}".format(link)) as ucursor: - for gis_row in ucursor: - ucursor.deleteRow() - logger.info("Disruption scenario removed OID {} from {} network".format(link, mode)) - del ucursor - line_num += 1 - - # double check the artificial links for intermodal facilities are set to 2 - # ------------------------------------------------------------------------- - ftot_supporting_gis.set_intermodal_links(the_scenario, logger) - - -# ============================================================================== - - -def import_afpat(logger, the_scenario): - # import the aftpat excel data to a table in the gdb - # -------------------------------------------------- - - ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) - - afpat_spreadsheet = os.path.join(ftot_program_directory, "lib", "AFPAT.xlsx") - - if not os.path.exists(afpat_spreadsheet): - error = "can't find afpat excel spreadsheet {}".format(afpat_spreadsheet) - raise IOError(error) - - import_afpat_data(afpat_spreadsheet, the_scenario.main_gdb, logger) - - ftot_supporting_gis.persist_AFPAT_tables(the_scenario, logger) - -# ============================================================================== - - -def create_main_db(logger, the_scenario): - # create a configuration table and a - # facilities table - #--------------------------------------------------- - - scenario_db = the_scenario.main_db - logger.info("start: create_main_db") - if os.path.exists(scenario_db): - logger.debug("start: deleting main.db") - logger.debug("sqlite file database {} exists".format(scenario_db)) - os.remove(scenario_db) - logger.debug("finished: deleted main.db") - - if not os.path.exists(scenario_db): - logger.debug("sqlite database file doesn't exist and will be created: {}".format(scenario_db)) - - with sqlite3.connect(scenario_db) as db_con: - - db_cur = db_con.cursor() - - db_cur.executescript( - "create table config(param text, value text, primary key(param));") - - logger.debug("finished: created main.db") - -# ============================================================================== + +# --------------------------------------------------------------------------------------------------- +# Name: ftot_setup +# +# Purpose: The ftot_setup module creates the main.db, and main.gdb for the scenario. The Alternative +# Fuel Production Assessment Tool (AFPAT) is also copied locally and stored in the main.gdb. +# +# --------------------------------------------------------------------------------------------------- + +import os +import datetime +import sqlite3 +from shutil import rmtree + +import ftot_supporting +import ftot_supporting_gis + +# =================================================================================================== + + +def import_afpat_data(afpat_spreadsheet, output_gdb, logger): + + import arcpy + + """ + import afpat from spreadsheet to arcgis table + """ + + logger.debug("start: import_afpat_data") + + if not os.path.exists(afpat_spreadsheet): + error = "can't find afpat spreadsheet {}".format(afpat_spreadsheet) + logger.error(error) + raise IOError(error) + + + if not os.path.exists(output_gdb): + + error = "can't find scratch gdb {}".format(output_gdb) + logger.error(error) + raise IOError(error) + + sheet = "2. User input & Results" + + out_table = os.path.join(output_gdb, "afpat_raw") + + if arcpy.Exists(out_table): + arcpy.Delete_management(out_table) + + arcpy.ExcelToTable_conversion(afpat_spreadsheet, out_table, sheet) + + logger.debug("finish: import_afpat_data") + + +# =================================================================================================== + +def cleanup(the_scenario, logger): + logger.info("start: cleanup") + + all_files = os.listdir(the_scenario.scenario_run_directory) + if all_files: + logger.info("deleting everything but the scenario .xml file and the .bat file.") + for file_or_dir in all_files: + + if file_or_dir.find("input_data") == -1: + if file_or_dir.find(".bat") == -1: + if file_or_dir.find(".xml") == -1: + try: + rmtree(os.path.join(the_scenario.scenario_run_directory, file_or_dir)) + except: + try: + os.remove(os.path.join(the_scenario.scenario_run_directory, file_or_dir)) + except: + pass + continue + all_files = os.listdir(the_scenario.scenario_run_directory) + if len(all_files) > 2: + logger.warning("something went wrong in checks_and_cleanup. There are {} extra folders and files".format(len(all_files)-2)) + else: + logger.debug("only two files left in the scenario run directory... continue") + +# =================================================================================================== + + +def setup(the_scenario, logger): + + logger.debug("start: setup") + start_time = datetime.datetime.now() + logger.info("Scenario Name: \t{}".format(the_scenario.scenario_name)) + logger.debug("Scenario Description: \t{}".format(the_scenario.scenario_description)) + logger.info("Scenario Start Date/Time: \t{}".format(start_time)) + + # create a folder for debug and intermediate files + # delete everything in there if it exists + # ------------------------------------------------ + debug_directory = os.path.join(the_scenario.scenario_run_directory, "debug") + + if os.path.exists(debug_directory): + logger.debug("deleting debug_directory and contents.") + rmtree(debug_directory) + + if not os.path.exists(debug_directory): + os.makedirs(debug_directory) + logger.debug("creating debug_directory.") + + # create the scenario database main.db + create_main_db(logger, the_scenario) + + # create the scenario geodatabase; main.gdb + create_main_gdb(logger, the_scenario) + + logger.debug("finish: SETUP: Runtime (HMS): \t{}".format(ftot_supporting.get_total_runtime_string(start_time))) + + +# =================================================================================================== + + +def create_main_gdb(logger, the_scenario): + # create the GIS geodatabase main.gdb + # ----------------------------------- + logger.info("start: create_main_gdb") + scenario_gdb = the_scenario.main_gdb + + import arcpy + + if arcpy.Exists(scenario_gdb): + + logger.debug("start: delete existing main.gdb") + logger.debug("main.gdb file location: " + scenario_gdb) + try: + arcpy.Delete_management(scenario_gdb) + + except: + logger.error("Couldn't delete " + scenario_gdb) + raise Exception("Couldn't delete " + scenario_gdb) + + # copy in the base network from the baseline data + # ----------------------------------------------- + + if not arcpy.Exists(the_scenario.base_network_gdb): + error = "can't find base network gdb {}".format(the_scenario.base_network_gdb) + raise IOError(error) + + logger.info("start: copy base network to main.gdb") + logger.config("Copy Scenario Base Network: \t{}".format(the_scenario.base_network_gdb)) + arcpy.Copy_management(the_scenario.base_network_gdb, scenario_gdb) + + # Check for disruption csv-- if one exists, this is where we remove links in the network that are fully disrupted. + if the_scenario.disruption_data == "None": + logger.info('disruption file not specified. No disruption to the network will be applied.') + else: + if not os.path.exists(the_scenario.disruption_data): + logger.warning("warning: cannot find disruption_data file: {}. No disruption to the network will be applied" + .format(the_scenario.disruption_data)) + else: + # check that is actually a csv + if not the_scenario.disruption_data.endswith("csv"): + error = "error: disruption_data file: {} is not a csv file. Please use valid disruption_data csv"\ + .format(the_scenario.disruption_data) + logger.error(error) + raise Exception(error) + else: + logger.info("Disruption csv to be applied to the network: {}".format(the_scenario.disruption_data)) + with open(the_scenario.disruption_data, 'r') as rf: + line_num = 1 + for line in rf: + csv_row = line.rstrip('\n').split(',') + if line_num == 1: + if csv_row[0] != 'mode' or csv_row[1]!= 'unique_link_id' or csv_row[2] != 'link_availability': + error = "Error: disruption_data file: {} does not match the appropriate disruption "\ + "data schema. Please check that the first three columns are 'mode', "\ + "'unique_link_id' and 'link_availability'".format(the_scenario.disruption_data) + logger.error(error) + raise Exception(error) + if line_num > 1: + mode = csv_row[0] + link = csv_row[1] + link_availability = float(csv_row[2]) + if link_availability == 0: # 0 = 100% disruption + with arcpy.da.UpdateCursor(os.path.join(scenario_gdb, "network/" + mode), ['OBJECTID'], "OBJECTID = {}".format(link)) as ucursor: + for gis_row in ucursor: + ucursor.deleteRow() + logger.info("Disruption scenario removed OID {} from {} network".format(link, mode)) + del ucursor + line_num += 1 + + # double check the artificial links for intermodal facilities are set to 2 + # ------------------------------------------------------------------------- + ftot_supporting_gis.set_intermodal_links(the_scenario, logger) + + +# ============================================================================== + + +def import_afpat(logger, the_scenario): + # import the aftpat excel data to a table in the gdb + # -------------------------------------------------- + + ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) + + afpat_spreadsheet = os.path.join(ftot_program_directory, "lib", "AFPAT.xlsx") + + if not os.path.exists(afpat_spreadsheet): + error = "can't find afpat excel spreadsheet {}".format(afpat_spreadsheet) + raise IOError(error) + + import_afpat_data(afpat_spreadsheet, the_scenario.main_gdb, logger) + + ftot_supporting_gis.persist_AFPAT_tables(the_scenario, logger) + +# ============================================================================== + + +def create_main_db(logger, the_scenario): + # create a configuration table and a + # facilities table + #--------------------------------------------------- + + scenario_db = the_scenario.main_db + logger.info("start: create_main_db") + if os.path.exists(scenario_db): + logger.debug("start: deleting main.db") + logger.debug("sqlite file database {} exists".format(scenario_db)) + os.remove(scenario_db) + logger.debug("finished: deleted main.db") + + if not os.path.exists(scenario_db): + logger.debug("sqlite database file doesn't exist and will be created: {}".format(scenario_db)) + + with sqlite3.connect(scenario_db) as db_con: + + db_cur = db_con.cursor() + + db_cur.executescript( + "create table config(param text, value text, primary key(param));") + + logger.debug("finished: created main.db") + +# ============================================================================== diff --git a/program/ftot_supporting.py b/program/ftot_supporting.py index 4dfc476..2ea63c7 100644 --- a/program/ftot_supporting.py +++ b/program/ftot_supporting.py @@ -1,637 +1,637 @@ -# --------------------------------------------------------------------------------------------------- -# Name: ftot_suppporting -# -# Purpose: -# -# --------------------------------------------------------------------------------------------------- -import os -import math - -import logging -import datetime -import sqlite3 -from ftot import ureg, Q_ -from six import iteritems - -# -def create_loggers(dirLocation, task): - """Create the logger""" - - loggingLocation = os.path.join(dirLocation, "logs") - - if not os.path.exists(loggingLocation): - os.makedirs(loggingLocation) - - # BELOW ARE THE LOGGING LEVELS. WHATEVER YOU CHOOSE IN SETLEVEL WILL BE SHOWN ALONG WITH HIGHER LEVELS. - # YOU CAN SET THIS FOR BOTH THE FILE LOG AND THE DOS WINDOW LOG - # ----------------------------------------------------------------------------------------------------- - # CRITICAL 50 - # ERROR 40 - # WARNING 30 - # RESULT 25 - # CONFIG 24 - # INFO 20 - # RUNTIME 11 - # DEBUG 10 - # DETAILED_DEBUG 5 - - logging.RESULT = 25 - logging.addLevelName(logging.RESULT, 'RESULT') - - logging.CONFIG = 19 - logging.addLevelName(logging.CONFIG, 'CONFIG') - - logging.RUNTIME = 11 - logging.addLevelName(logging.RUNTIME, 'RUNTIME') - - logging.DETAILED_DEBUG = 5 - logging.addLevelName(logging.DETAILED_DEBUG, 'DETAILED_DEBUG') - - logger = logging.getLogger('log') - logger.setLevel(logging.DEBUG) - - logger.runtime = lambda msg, *args: logger._log(logging.RUNTIME, msg, args) - logger.result = lambda msg, *args: logger._log(logging.RESULT, msg, args) - logger.config = lambda msg, *args: logger._log(logging.CONFIG, msg, args) - logger.detailed_debug = lambda msg, *args: logger._log(logging.DETAILED_DEBUG, msg, args) - - # FILE LOG - # ------------------------------------------------------------------------------ - logFileName = task + "_" + "log_" + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + ".log" - file_log = logging.FileHandler(os.path.join(loggingLocation, logFileName), mode='a') - - file_log.setLevel(logging.DEBUG) - - - file_log_format = logging.Formatter('%(asctime)s.%(msecs).03d %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M:%S') - file_log.setFormatter(file_log_format) - - # DOS WINDOW LOG - # ------------------------------------------------------------------------------ - console = logging.StreamHandler() - - console.setLevel(logging.INFO) - - console_log_format = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M:%S') - console.setFormatter(console_log_format) - - # ADD THE HANDLERS - # ---------------- - logger.addHandler(file_log) - logger.addHandler(console) - - # NOTE: with these custom levels you can now do the following - # test this out once the handlers have been added - # ------------------------------------------------------------ - - return logger - - -# ============================================================================== - - -def clean_file_name(value): - deletechars = '\/:*?"<>|' - for c in deletechars: - value = value.replace(c, '') - return value; - - -# ============================================================================== - - -def get_total_runtime_string(start_time): - end_time = datetime.datetime.now() - - duration = end_time - start_time - - seconds = duration.total_seconds() - - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - seconds = int(seconds % 60) - - hms = str("{:02}:{:02}:{:02}").format(hours, minutes, seconds) - - return hms - - -# ============================================================================== - -def euclidean_distance(xCoord, yCoord, xCoord2, yCoord2): - """Calculates the Euclidean distance between two sets of coordinates""" - - return math.sqrt(math.pow((xCoord - xCoord2), 2) + math.pow((yCoord - yCoord2), 2)) - - -# ============================================================================= - -class CropData: - """Class object containing crop information""" - - def __init__(self, prod, crop): - self.production = prod - self.crop = crop - - -# ============================================================================== -def check_OD_commodities_for_routes(origin_commodity_slate, destination_commodity_slate, logger): - # check commodities and see if they match - - OD_commodity_match_dict = {} - - all_origin_commodities = origin_commodity_slate.commodities - - for origin_commodity in all_origin_commodities: - - all_destination_commodities = destination_commodity_slate.commodities - - for destination_commodity in all_destination_commodities: - - if origin_commodity.find(destination_commodity) > -1: - - OD_commodity_match_dict[origin_commodity] = all_origin_commodities[origin_commodity] - - return OD_commodity_match_dict - - -# ============================================================================== - -def split_feedstock_commidity_name_into_parts(feedstock, logger): - # Source Categories - afpat_source_category_list = ["Agricultural Residues", "Woody Crops and Residues", "Herbaceous Energy Crops", - "Oil Crops", "Waste Oils and Animal Fats", "Sugary and Starchy Biomass", "Bakken", - "None"] - feedstock_in_parts = () - feedstock_type = "" - the_source_category = "" - feedstock_source = "" - - for source_category in afpat_source_category_list: - - # add "_" to the front and back of the string so the type and source will be cleanly split - source_category_as_separator = str("_" + source_category.replace(" ", "_") + "_") - - # Split the string at the first occurrence of sep, and return a 3-tuple - # containing the part before the separator, the separator itself, and - # the part after the separator. - feedstock_in_parts = feedstock.partition(source_category_as_separator) - - # If the separator is not found, return a 3-tuple containing the string itself, - # followed by two empty strings. - if not feedstock_in_parts[2] == "": - feedstock_type = feedstock_in_parts[0] - - the_source_category = source_category.replace(" ", "_").replace("-", "_") - - feedstock_source = feedstock_in_parts[2] - if feedstock_type == "": - logger.warning("the feedstock {} is not in the list {}".format(feedstock, afpat_source_category_list)) - logger.error("did not parse the feedstock commodity name into parts") - raise Exception("did not parse the feedstock commodity name into parts") - - return [feedstock_type, the_source_category, feedstock_source] - - -# ============================================================================== - -def create_full_crop_name(Feedstock_Type, Source_Category, Feedstock_Source): - crop = str(Feedstock_Type + "_" + Source_Category + "_" + Feedstock_Source).replace(" ", "_").replace("-", "_") - - return crop - - -# ============================================================================== - -def get_cleaned_process_name(_Primary_Processing_Type, _Secondary_Processing_Type, _Tertiary_Processing_Type): - Primary_Processing_Type = str(_Primary_Processing_Type).replace(" ", "_").replace("-", "_") - Secondary_Processing_Type = str(_Secondary_Processing_Type).replace(" ", "_").replace("-", "_") - Tertiary_Processing_Type = str(_Tertiary_Processing_Type).replace(" ", "_").replace("-", "_") - - return (Primary_Processing_Type, Secondary_Processing_Type, Tertiary_Processing_Type) - - -# ============================================================================== -def make_rmp_as_proc_slate(the_scenario, commodity_name, commodity_quantity_with_units, logger): - # we're going to query the database for facilities named candidate* (wildcard) - # and use their product slate ratio to return a fuel_dictrionary. - sql = """ select f.facility_name, c.commodity_name, fc.quantity, fc.units, c.phase_of_matter, fc.io - from facility_commodities fc - join facilities f on f.facility_id = fc.facility_id - join commodities c on c.commodity_id = fc.commodity_id - where facility_name like 'candidate%' - """ - - input_commodities = {} - output_commodities = {} - scaled_output_dict = {} - with sqlite3.connect(the_scenario.main_db) as db_con: - - db_cur = db_con.cursor() - db_cur.execute(sql) - - for row in db_cur: - facility_name = row[0] - a_commodity_name = row[1] - quantity = row[2] - units = row[3] - phase_of_matter = row[4] - io = row[5] - - if io == 'i': - if not facility_name in list(input_commodities.keys()): - input_commodities[facility_name] = [] - - input_commodities[facility_name].append([a_commodity_name, quantity, units, phase_of_matter, io]) - - elif io == 'o': - if not facility_name in list(output_commodities.keys()): - output_commodities[facility_name] = [] - output_commodities[facility_name].append([a_commodity_name, quantity, units, phase_of_matter, io]) - elif io == 'maxsize' or io == 'minsize': - logger.detailed_debug("io flag == maxsize or min size") - elif io == 'cost_formula': - logger.detailed_debug("io flag == cost_formula") - else: - logger.warning( - "the io flag: {} is not recognized for commodity: {} - at facility: {}".format(io, a_commodity_name, - facility_name)) - - # check if there is more than one input commodity - for facility in input_commodities: - if not len(input_commodities[facility]) == 1: - logger.warning( - "there are: {} input commodities in the product slate for facility {}".format(len(input_commodities), - facility_name)) - for an_input_commodity in input_commodities[facility]: - logger.warning("commodity_name: {}, quantity: {}, units: {}, io: {}".format(an_input_commodity[0], - an_input_commodity[1], - an_input_commodity[2], - an_input_commodity[3])) - else: # there is only one input commodity to use in the ratio. - - # check if this is the ratio we want to save - if input_commodities[facility][0][0].lower() == commodity_name.lower(): - - a_commodity_name = input_commodities[facility][0][0] - quantity = input_commodities[facility][0][1] - units = input_commodities[facility][0][2] - input_commodity_quantity_with_units = Q_(quantity, units) - - # store all the output commodities - for an_output_commodity in output_commodities[facility_name]: - a_commodity_name = an_output_commodity[0] - quantity = an_output_commodity[1] - units = an_output_commodity[2] - phase_of_matter = an_output_commodity[3] - output_commodity_quantity_with_units = Q_(quantity, units) - - # the output commodity quantity is divided by the input - # commodity quantity specified in the candidate slate csv - # and then multiplied by the commodity_quantity_with_units - # factor from the RMP passed into this module - oc = output_commodity_quantity_with_units - ic = input_commodity_quantity_with_units - cs = commodity_quantity_with_units - - # finally add it to the scaled output dictionary - scaled_output_dict[a_commodity_name] = [(oc / ic * cs), phase_of_matter] - - return scaled_output_dict - - -# ============================================================================== -def get_max_fuel_conversion_process_for_commodity(commodity, the_scenario, logger): - - max_conversion_process = "" # max conversion process - processes_for_commodity = [] # list of all processess for feedstock - conversion_efficiency_dict = {} # a temporary dictionary to store the conversion efficiencies for each process - - ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) - - if commodity in ag_fuel_yield_dict: - processes_for_commodity.extend(list(ag_fuel_yield_dict[commodity].keys())) - - # DO THE BIOWASTE RESOURCES - if commodity in bioWasteDict: - processes_for_commodity.extend(list(bioWasteDict[commodity].keys())) - - # DO THE FOSSIL RESOURCES - fossil_keys = list(fossilResources.keys()) - - for key in fossil_keys: - - if commodity.find(key) > -1: - processes_for_commodity.extend(list(fossilResources[key].keys())) - - if processes_for_commodity == []: - logger.warning("processes_for_commodity is empty: {}".format(processes_for_commodity)) - - for conversion_process in processes_for_commodity: - input_commodity, output_commodity = get_input_and_output_commodity_quantities_from_afpat(commodity, - conversion_process, - the_scenario, logger) - - conversion_efficiency = input_commodity / output_commodity["total_fuel"] - - conversion_efficiency_dict[conversion_process] = conversion_efficiency - - # get the most efficient conversion process from the sorted conversion_efficiency_dict - # this method is some black magic using the lambda keyword to sort the dictionary by - # values and gets the lowest kg/bbl of total fuel conversion rate (== max biomass -> fuel efficiency) - if conversion_efficiency_dict == {}: - logger.warning("conversion_efficiency_dict is empty: {}".format(conversion_efficiency_dict)) - - else: - max_conversion_process = sorted(iteritems(conversion_efficiency_dict), key=lambda k_v: (k_v[1], k_v[0]))[0] - - return max_conversion_process # just return the name of the max conversion process, not the conversion efficiency - - -# ============================================================================== - -def create_list_of_sub_commodities_from_afpat(commodity, process, the_scenario, logger): - ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) - - list_of_sub_commodities = [] - - # first check the agricultural feedstocks in the ag_fuel_dict - - for ag_key in ag_fuel_yield_dict: - - if ag_key.lower().find(commodity) > -1: - logger.debug("adding commodity {} list_of_sub_commodities for {}".format(ag_key, commodity)) - list_of_sub_commodities.append(ag_key) - - # second check the biowaste resources in the biowaste dict - - for biowaste_key in bioWasteDict: - - if biowaste_key.lower().find(commodity) > -1: - logger.debug("adding commodity {} list_of_sub_commodities for {}".format(biowaste_key, commodity)) - list_of_sub_commodities.append(biowaste_key) - - # last, check the fossil resources feedstocks in the fossil resource dict - - for fossil_key in fossilResources: - - if fossil_key.lower().find(commodity) > -1: - logger.debug("adding commodity {} list_of_sub_commodities for {}".format(fossil_key, commodity)) - list_of_sub_commodities.append(fossil_key) - - if list_of_sub_commodities == []: - list_of_sub_commodities = [ - "the commodity {} has no sub_commodities in AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( - commodity)] - logger.warning( - "the commodity {} has no sub_commodities in AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( - commodity)) - - return list_of_sub_commodities - - -# ============================================================================== -def get_demand_met_multiplier(simple_fuel_name, primary_process_type, logger): - # switch the demand_met_multiplier based on fuel name and process type - if simple_fuel_name == "jet": - if primary_process_type == "HEFA": - demand_met_multiplier = 2 - elif primary_process_type == "FTx": - demand_met_multiplier = 2 - elif primary_process_type == "AFx": - # 30% alcohol/fuel to 70% petroleum - demand_met_multiplier = float((3 + 1 / 3)) - elif primary_process_type == "Petroleum_Refinery": - demand_met_multiplier = 1 - elif primary_process_type == "NA": - demand_met_multiplier = 1 - else: - logger.error("the demand met multiplier was not set") - raise Exception("the demand met multiplier was not set") - - elif simple_fuel_name == "diesel": - - demand_met_multiplier = 1 - - else: - demand_met_multiplier = 1 - logger.error("the fuel type is not jet or diesel; demand met multiplier was set to 1") - - return demand_met_multiplier - - -# ============================================================================== - -# ============================================================================== -def get_processor_capacity(primary_processing, logger): - capacity = 0 - - if primary_processing == "FTx": - capacity = Q_(200000, "kgal") - if primary_processing == "Petroleum_Refinery": - capacity = Q_(7665000, "kgal") - else: - capacity = Q_(200000, "kgal") - - return capacity - - -# ============================================================================== - -def load_afpat_tables(the_scenario, logger): - import pickle - pickle_file = os.path.join(the_scenario.scenario_run_directory, "debug", "AFPAT_tables.p") - - afpat_tables = pickle.load(open(pickle_file, "rb")) - ag_fuel_yield_dict = afpat_tables[0] - cropYield = afpat_tables[1] - bioWasteDict = afpat_tables[2] - fossilResources = afpat_tables[3] - - return ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources - - -# ============================================================================== - -def get_input_and_output_commodity_quantities_from_afpat(commodity, process, the_scenario, logger): - - input_commodity_quantities = 0 # a quantity of input resource required to produce fuels - output_commodity_quantities = {} # a dictionary containing the fuel outputs - - ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) - - if commodity.lower().find("test_liquid_none_none") > -1: - # print "in the right place" - input_commodity_quantities = Q_(1, "kgal") - output_commodity_quantities['test_product_liquid_None_None'] = Q_(1, "kgal") - output_commodity_quantities['jet'] = Q_(0, "kgal") - output_commodity_quantities['diesel'] = Q_(0, "kgal") - output_commodity_quantities['naphtha'] = Q_(0, "kgal") - output_commodity_quantities['aromatics'] = Q_(0, "kgal") - output_commodity_quantities['total_fuel'] = Q_(1, "kgal") - # hack to skip the petroleum refinery - commodity = "hack-hack-hack" - process = "hack-hack-hack" - - elif commodity in ag_fuel_yield_dict: - # print "in the wrong right place" - if process in ag_fuel_yield_dict[commodity]: - - input_commodity_quantities = Q_(ag_fuel_yield_dict[commodity][process][8], "kg/day") - - output_commodity_quantities['jet'] = Q_(ag_fuel_yield_dict[commodity][process][1], "oil_bbl / day") - output_commodity_quantities['diesel'] = Q_(ag_fuel_yield_dict[commodity][process][2], "oil_bbl / day") - output_commodity_quantities['naphtha'] = Q_(ag_fuel_yield_dict[commodity][process][3], "oil_bbl / day") - output_commodity_quantities['aromatics'] = Q_(ag_fuel_yield_dict[commodity][process][4], "oil_bbl / day") - output_commodity_quantities['total_fuel'] = Q_(ag_fuel_yield_dict[commodity][process][6], "oil_bbl / day") - - else: - - logger.error( - "the commodity {} has no process {} in the AFPAT agricultural yield dictionary".format(commodity, - process)) - raise Exception( - "the commodity {} has no process {} in the AFPAT agricultural yield dictionary".format(commodity, - process)) - - elif commodity in bioWasteDict: - - if process in bioWasteDict[commodity]: - - input_commodity_quantities = Q_(bioWasteDict[commodity][process][1], "kg / year") - - output_commodity_quantities['total_fuel'] = Q_(bioWasteDict[commodity][process][4], "oil_bbl / day") - output_commodity_quantities['jet'] = Q_(bioWasteDict[commodity][process][5], "oil_bbl / day") - output_commodity_quantities['diesel'] = Q_(bioWasteDict[commodity][process][6], "oil_bbl / day") - output_commodity_quantities['naphtha'] = Q_(bioWasteDict[commodity][process][7], "oil_bbl / day") - output_commodity_quantities['aromatics'] = Q_(0.000, "oil_bbl / day") - - else: - - logger.debug( - "the process {} for commodity {} is not in the biowaste yield dictionary {}".format(process, commodity, - bioWasteDict[ - commodity])) - logger.error( - "the commodity {} has no process {} in the AFPAT biowaste yield dictionary".format(commodity, process)) - raise Exception( - "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, - process)) - - # DO THE FOSSIL RESOURCES - fossil_keys = list(fossilResources.keys()) - - for key in fossil_keys: - - if commodity.find(key) > -1: - - if process in fossilResources[key]: - - input_commodity_quantities = Q_(500e3, "oil_bbl / day") - - output_commodity_quantities['total_fuel'] = Q_(fossilResources[key][process][4], "oil_bbl / day") - output_commodity_quantities['jet'] = Q_(fossilResources[key][process][5], "oil_bbl / day") - output_commodity_quantities['diesel'] = Q_(fossilResources[key][process][6], "oil_bbl / day") - output_commodity_quantities['naphtha'] = Q_(fossilResources[key][process][7], "oil_bbl / day") - output_commodity_quantities['aromatics'] = Q_(0.000, "oil_bbl / day") - - else: - - logger.error( - "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, - process)) - raise Exception( - "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, - process)) - - if input_commodity_quantities == 0 or output_commodity_quantities == {}: - logger.error( - "the commodity {} and process {} is not in the AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( - commodity, process)) - raise Exception( - "the commodity {} and process {} is not in the AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( - commodity, process)) - - return input_commodity_quantities, output_commodity_quantities - - -# ============================================================================= -def get_RMP_commodity_list(the_scenario, logger): - import sqlite3 - - main_db = the_scenario.main_db - RMP_commodity_list = [] - - with sqlite3.connect(main_db) as db_con: - db_cur = db_con.cursor() - - sql = "select distinct commodity from raw_material_producers;" - db_cur.execute(sql) - - for record in db_cur: - RMP_commodity_list.append(record[0]) - - return RMP_commodity_list - - -# ============================================================================== -def get_route_type(commodity_name, rmp_commodity_list): - route_type = "" - - if commodity_name in rmp_commodity_list: - - route_type = "RMP_Processor" - - else: - - route_type = "Processor_Destination" - - return route_type - - -# ============================================================================== - -def get_commodity_simple_name(commodity_name): - simple_commodity_name = "" - - if commodity_name.find("diesel") >= 0: - - simple_commodity_name = "diesel" - - elif commodity_name.find("jet") >= 0: - - simple_commodity_name = "jet" - - else: - - simple_commodity_name = commodity_name # a raw material producer material that doesn't have a simple name - - return simple_commodity_name - - -# ============================================================================== -def post_optimization(the_scenario, task_id, logger): - from ftot_pulp import parse_optimal_solution_db - - logger.info("START: post_optimization for {} task".format(task_id)) - start_time = datetime.datetime.now() - - # Parse the Problem for the Optimal Solution - parsed_optimal_solution = parse_optimal_solution_db(the_scenario, logger) - - logger.info("FINISH: post_optimization: Runtime (HMS): \t{}".format(get_total_runtime_string(start_time))) - -# =================================================================================================== - - -def debug_write_solution(the_scenario, prob, logfile, logger): - logger.info("starting debug_write_solution") - - with open(os.path.join(the_scenario.scenario_run_directory, "logs", logfile), 'w') as solution_file: - - for v in prob.variables(): - - if v.varValue > 0.0: - solution_file.write("{} -> {:,.2f}\n".format(v.name, v.varValue)) - logger.info("finished: debug_write_solution") +# --------------------------------------------------------------------------------------------------- +# Name: ftot_suppporting +# +# Purpose: +# +# --------------------------------------------------------------------------------------------------- +import os +import math + +import logging +import datetime +import sqlite3 +from ftot import ureg, Q_ +from six import iteritems + +# +def create_loggers(dirLocation, task): + """Create the logger""" + + loggingLocation = os.path.join(dirLocation, "logs") + + if not os.path.exists(loggingLocation): + os.makedirs(loggingLocation) + + # BELOW ARE THE LOGGING LEVELS. WHATEVER YOU CHOOSE IN SETLEVEL WILL BE SHOWN ALONG WITH HIGHER LEVELS. + # YOU CAN SET THIS FOR BOTH THE FILE LOG AND THE DOS WINDOW LOG + # ----------------------------------------------------------------------------------------------------- + # CRITICAL 50 + # ERROR 40 + # WARNING 30 + # RESULT 25 + # CONFIG 24 + # INFO 20 + # RUNTIME 11 + # DEBUG 10 + # DETAILED_DEBUG 5 + + logging.RESULT = 25 + logging.addLevelName(logging.RESULT, 'RESULT') + + logging.CONFIG = 19 + logging.addLevelName(logging.CONFIG, 'CONFIG') + + logging.RUNTIME = 11 + logging.addLevelName(logging.RUNTIME, 'RUNTIME') + + logging.DETAILED_DEBUG = 5 + logging.addLevelName(logging.DETAILED_DEBUG, 'DETAILED_DEBUG') + + logger = logging.getLogger('log') + logger.setLevel(logging.DEBUG) + + logger.runtime = lambda msg, *args: logger._log(logging.RUNTIME, msg, args) + logger.result = lambda msg, *args: logger._log(logging.RESULT, msg, args) + logger.config = lambda msg, *args: logger._log(logging.CONFIG, msg, args) + logger.detailed_debug = lambda msg, *args: logger._log(logging.DETAILED_DEBUG, msg, args) + + # FILE LOG + # ------------------------------------------------------------------------------ + logFileName = task + "_" + "log_" + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + ".log" + file_log = logging.FileHandler(os.path.join(loggingLocation, logFileName), mode='a') + + file_log.setLevel(logging.DEBUG) + + + file_log_format = logging.Formatter('%(asctime)s.%(msecs).03d %(levelname)-8s %(message)s', + datefmt='%m-%d %H:%M:%S') + file_log.setFormatter(file_log_format) + + # DOS WINDOW LOG + # ------------------------------------------------------------------------------ + console = logging.StreamHandler() + + console.setLevel(logging.INFO) + + console_log_format = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s', datefmt='%m-%d %H:%M:%S') + console.setFormatter(console_log_format) + + # ADD THE HANDLERS + # ---------------- + logger.addHandler(file_log) + logger.addHandler(console) + + # NOTE: with these custom levels you can now do the following + # test this out once the handlers have been added + # ------------------------------------------------------------ + + return logger + + +# ============================================================================== + + +def clean_file_name(value): + deletechars = '\/:*?"<>|' + for c in deletechars: + value = value.replace(c, '') + return value; + + +# ============================================================================== + + +def get_total_runtime_string(start_time): + end_time = datetime.datetime.now() + + duration = end_time - start_time + + seconds = duration.total_seconds() + + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + seconds = int(seconds % 60) + + hms = str("{:02}:{:02}:{:02}").format(hours, minutes, seconds) + + return hms + + +# ============================================================================== + +def euclidean_distance(xCoord, yCoord, xCoord2, yCoord2): + """Calculates the Euclidean distance between two sets of coordinates""" + + return math.sqrt(math.pow((xCoord - xCoord2), 2) + math.pow((yCoord - yCoord2), 2)) + + +# ============================================================================= + +class CropData: + """Class object containing crop information""" + + def __init__(self, prod, crop): + self.production = prod + self.crop = crop + + +# ============================================================================== +def check_OD_commodities_for_routes(origin_commodity_slate, destination_commodity_slate, logger): + # check commodities and see if they match + + OD_commodity_match_dict = {} + + all_origin_commodities = origin_commodity_slate.commodities + + for origin_commodity in all_origin_commodities: + + all_destination_commodities = destination_commodity_slate.commodities + + for destination_commodity in all_destination_commodities: + + if origin_commodity.find(destination_commodity) > -1: + + OD_commodity_match_dict[origin_commodity] = all_origin_commodities[origin_commodity] + + return OD_commodity_match_dict + + +# ============================================================================== + +def split_feedstock_commidity_name_into_parts(feedstock, logger): + # Source Categories + afpat_source_category_list = ["Agricultural Residues", "Woody Crops and Residues", "Herbaceous Energy Crops", + "Oil Crops", "Waste Oils and Animal Fats", "Sugary and Starchy Biomass", "Bakken", + "None"] + feedstock_in_parts = () + feedstock_type = "" + the_source_category = "" + feedstock_source = "" + + for source_category in afpat_source_category_list: + + # add "_" to the front and back of the string so the type and source will be cleanly split + source_category_as_separator = str("_" + source_category.replace(" ", "_") + "_") + + # Split the string at the first occurrence of sep, and return a 3-tuple + # containing the part before the separator, the separator itself, and + # the part after the separator. + feedstock_in_parts = feedstock.partition(source_category_as_separator) + + # If the separator is not found, return a 3-tuple containing the string itself, + # followed by two empty strings. + if not feedstock_in_parts[2] == "": + feedstock_type = feedstock_in_parts[0] + + the_source_category = source_category.replace(" ", "_").replace("-", "_") + + feedstock_source = feedstock_in_parts[2] + if feedstock_type == "": + logger.warning("the feedstock {} is not in the list {}".format(feedstock, afpat_source_category_list)) + logger.error("did not parse the feedstock commodity name into parts") + raise Exception("did not parse the feedstock commodity name into parts") + + return [feedstock_type, the_source_category, feedstock_source] + + +# ============================================================================== + +def create_full_crop_name(Feedstock_Type, Source_Category, Feedstock_Source): + crop = str(Feedstock_Type + "_" + Source_Category + "_" + Feedstock_Source).replace(" ", "_").replace("-", "_") + + return crop + + +# ============================================================================== + +def get_cleaned_process_name(_Primary_Processing_Type, _Secondary_Processing_Type, _Tertiary_Processing_Type): + Primary_Processing_Type = str(_Primary_Processing_Type).replace(" ", "_").replace("-", "_") + Secondary_Processing_Type = str(_Secondary_Processing_Type).replace(" ", "_").replace("-", "_") + Tertiary_Processing_Type = str(_Tertiary_Processing_Type).replace(" ", "_").replace("-", "_") + + return (Primary_Processing_Type, Secondary_Processing_Type, Tertiary_Processing_Type) + + +# ============================================================================== +def make_rmp_as_proc_slate(the_scenario, commodity_name, commodity_quantity_with_units, logger): + # we're going to query the database for facilities named candidate* (wildcard) + # and use their product slate ratio to return a fuel_dictrionary. + sql = """ select f.facility_name, c.commodity_name, fc.quantity, fc.units, c.phase_of_matter, fc.io + from facility_commodities fc + join facilities f on f.facility_id = fc.facility_id + join commodities c on c.commodity_id = fc.commodity_id + where facility_name like 'candidate%' + """ + + input_commodities = {} + output_commodities = {} + scaled_output_dict = {} + with sqlite3.connect(the_scenario.main_db) as db_con: + + db_cur = db_con.cursor() + db_cur.execute(sql) + + for row in db_cur: + facility_name = row[0] + a_commodity_name = row[1] + quantity = row[2] + units = row[3] + phase_of_matter = row[4] + io = row[5] + + if io == 'i': + if not facility_name in list(input_commodities.keys()): + input_commodities[facility_name] = [] + + input_commodities[facility_name].append([a_commodity_name, quantity, units, phase_of_matter, io]) + + elif io == 'o': + if not facility_name in list(output_commodities.keys()): + output_commodities[facility_name] = [] + output_commodities[facility_name].append([a_commodity_name, quantity, units, phase_of_matter, io]) + elif io == 'maxsize' or io == 'minsize': + logger.detailed_debug("io flag == maxsize or min size") + elif io == 'cost_formula': + logger.detailed_debug("io flag == cost_formula") + else: + logger.warning( + "the io flag: {} is not recognized for commodity: {} - at facility: {}".format(io, a_commodity_name, + facility_name)) + + # check if there is more than one input commodity + for facility in input_commodities: + if not len(input_commodities[facility]) == 1: + logger.warning( + "there are: {} input commodities in the product slate for facility {}".format(len(input_commodities), + facility_name)) + for an_input_commodity in input_commodities[facility]: + logger.warning("commodity_name: {}, quantity: {}, units: {}, io: {}".format(an_input_commodity[0], + an_input_commodity[1], + an_input_commodity[2], + an_input_commodity[3])) + else: # there is only one input commodity to use in the ratio. + + # check if this is the ratio we want to save + if input_commodities[facility][0][0].lower() == commodity_name.lower(): + + a_commodity_name = input_commodities[facility][0][0] + quantity = input_commodities[facility][0][1] + units = input_commodities[facility][0][2] + input_commodity_quantity_with_units = Q_(quantity, units) + + # store all the output commodities + for an_output_commodity in output_commodities[facility_name]: + a_commodity_name = an_output_commodity[0] + quantity = an_output_commodity[1] + units = an_output_commodity[2] + phase_of_matter = an_output_commodity[3] + output_commodity_quantity_with_units = Q_(quantity, units) + + # the output commodity quantity is divided by the input + # commodity quantity specified in the candidate slate csv + # and then multiplied by the commodity_quantity_with_units + # factor from the RMP passed into this module + oc = output_commodity_quantity_with_units + ic = input_commodity_quantity_with_units + cs = commodity_quantity_with_units + + # finally add it to the scaled output dictionary + scaled_output_dict[a_commodity_name] = [(oc / ic * cs), phase_of_matter] + + return scaled_output_dict + + +# ============================================================================== +def get_max_fuel_conversion_process_for_commodity(commodity, the_scenario, logger): + + max_conversion_process = "" # max conversion process + processes_for_commodity = [] # list of all processess for feedstock + conversion_efficiency_dict = {} # a temporary dictionary to store the conversion efficiencies for each process + + ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) + + if commodity in ag_fuel_yield_dict: + processes_for_commodity.extend(list(ag_fuel_yield_dict[commodity].keys())) + + # DO THE BIOWASTE RESOURCES + if commodity in bioWasteDict: + processes_for_commodity.extend(list(bioWasteDict[commodity].keys())) + + # DO THE FOSSIL RESOURCES + fossil_keys = list(fossilResources.keys()) + + for key in fossil_keys: + + if commodity.find(key) > -1: + processes_for_commodity.extend(list(fossilResources[key].keys())) + + if processes_for_commodity == []: + logger.warning("processes_for_commodity is empty: {}".format(processes_for_commodity)) + + for conversion_process in processes_for_commodity: + input_commodity, output_commodity = get_input_and_output_commodity_quantities_from_afpat(commodity, + conversion_process, + the_scenario, logger) + + conversion_efficiency = input_commodity / output_commodity["total_fuel"] + + conversion_efficiency_dict[conversion_process] = conversion_efficiency + + # get the most efficient conversion process from the sorted conversion_efficiency_dict + # this method is some black magic using the lambda keyword to sort the dictionary by + # values and gets the lowest kg/bbl of total fuel conversion rate (== max biomass -> fuel efficiency) + if conversion_efficiency_dict == {}: + logger.warning("conversion_efficiency_dict is empty: {}".format(conversion_efficiency_dict)) + + else: + max_conversion_process = sorted(iteritems(conversion_efficiency_dict), key=lambda k_v: (k_v[1], k_v[0]))[0] + + return max_conversion_process # just return the name of the max conversion process, not the conversion efficiency + + +# ============================================================================== + +def create_list_of_sub_commodities_from_afpat(commodity, process, the_scenario, logger): + ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) + + list_of_sub_commodities = [] + + # first check the agricultural feedstocks in the ag_fuel_dict + + for ag_key in ag_fuel_yield_dict: + + if ag_key.lower().find(commodity) > -1: + logger.debug("adding commodity {} list_of_sub_commodities for {}".format(ag_key, commodity)) + list_of_sub_commodities.append(ag_key) + + # second check the biowaste resources in the biowaste dict + + for biowaste_key in bioWasteDict: + + if biowaste_key.lower().find(commodity) > -1: + logger.debug("adding commodity {} list_of_sub_commodities for {}".format(biowaste_key, commodity)) + list_of_sub_commodities.append(biowaste_key) + + # last, check the fossil resources feedstocks in the fossil resource dict + + for fossil_key in fossilResources: + + if fossil_key.lower().find(commodity) > -1: + logger.debug("adding commodity {} list_of_sub_commodities for {}".format(fossil_key, commodity)) + list_of_sub_commodities.append(fossil_key) + + if list_of_sub_commodities == []: + list_of_sub_commodities = [ + "the commodity {} has no sub_commodities in AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( + commodity)] + logger.warning( + "the commodity {} has no sub_commodities in AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( + commodity)) + + return list_of_sub_commodities + + +# ============================================================================== +def get_demand_met_multiplier(simple_fuel_name, primary_process_type, logger): + # switch the demand_met_multiplier based on fuel name and process type + if simple_fuel_name == "jet": + if primary_process_type == "HEFA": + demand_met_multiplier = 2 + elif primary_process_type == "FTx": + demand_met_multiplier = 2 + elif primary_process_type == "AFx": + # 30% alcohol/fuel to 70% petroleum + demand_met_multiplier = float((3 + 1 / 3)) + elif primary_process_type == "Petroleum_Refinery": + demand_met_multiplier = 1 + elif primary_process_type == "NA": + demand_met_multiplier = 1 + else: + logger.error("the demand met multiplier was not set") + raise Exception("the demand met multiplier was not set") + + elif simple_fuel_name == "diesel": + + demand_met_multiplier = 1 + + else: + demand_met_multiplier = 1 + logger.error("the fuel type is not jet or diesel; demand met multiplier was set to 1") + + return demand_met_multiplier + + +# ============================================================================== + +# ============================================================================== +def get_processor_capacity(primary_processing, logger): + capacity = 0 + + if primary_processing == "FTx": + capacity = Q_(200000, "kgal") + if primary_processing == "Petroleum_Refinery": + capacity = Q_(7665000, "kgal") + else: + capacity = Q_(200000, "kgal") + + return capacity + + +# ============================================================================== + +def load_afpat_tables(the_scenario, logger): + import pickle + pickle_file = os.path.join(the_scenario.scenario_run_directory, "debug", "AFPAT_tables.p") + + afpat_tables = pickle.load(open(pickle_file, "rb")) + ag_fuel_yield_dict = afpat_tables[0] + cropYield = afpat_tables[1] + bioWasteDict = afpat_tables[2] + fossilResources = afpat_tables[3] + + return ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources + + +# ============================================================================== + +def get_input_and_output_commodity_quantities_from_afpat(commodity, process, the_scenario, logger): + + input_commodity_quantities = 0 # a quantity of input resource required to produce fuels + output_commodity_quantities = {} # a dictionary containing the fuel outputs + + ag_fuel_yield_dict, cropYield, bioWasteDict, fossilResources = load_afpat_tables(the_scenario, logger) + + if commodity.lower().find("test_liquid_none_none") > -1: + # print "in the right place" + input_commodity_quantities = Q_(1, "kgal") + output_commodity_quantities['test_product_liquid_None_None'] = Q_(1, "kgal") + output_commodity_quantities['jet'] = Q_(0, "kgal") + output_commodity_quantities['diesel'] = Q_(0, "kgal") + output_commodity_quantities['naphtha'] = Q_(0, "kgal") + output_commodity_quantities['aromatics'] = Q_(0, "kgal") + output_commodity_quantities['total_fuel'] = Q_(1, "kgal") + # hack to skip the petroleum refinery + commodity = "hack-hack-hack" + process = "hack-hack-hack" + + elif commodity in ag_fuel_yield_dict: + # print "in the wrong right place" + if process in ag_fuel_yield_dict[commodity]: + + input_commodity_quantities = Q_(ag_fuel_yield_dict[commodity][process][8], "kg/day") + + output_commodity_quantities['jet'] = Q_(ag_fuel_yield_dict[commodity][process][1], "oil_bbl / day") + output_commodity_quantities['diesel'] = Q_(ag_fuel_yield_dict[commodity][process][2], "oil_bbl / day") + output_commodity_quantities['naphtha'] = Q_(ag_fuel_yield_dict[commodity][process][3], "oil_bbl / day") + output_commodity_quantities['aromatics'] = Q_(ag_fuel_yield_dict[commodity][process][4], "oil_bbl / day") + output_commodity_quantities['total_fuel'] = Q_(ag_fuel_yield_dict[commodity][process][6], "oil_bbl / day") + + else: + + logger.error( + "the commodity {} has no process {} in the AFPAT agricultural yield dictionary".format(commodity, + process)) + raise Exception( + "the commodity {} has no process {} in the AFPAT agricultural yield dictionary".format(commodity, + process)) + + elif commodity in bioWasteDict: + + if process in bioWasteDict[commodity]: + + input_commodity_quantities = Q_(bioWasteDict[commodity][process][1], "kg / year") + + output_commodity_quantities['total_fuel'] = Q_(bioWasteDict[commodity][process][4], "oil_bbl / day") + output_commodity_quantities['jet'] = Q_(bioWasteDict[commodity][process][5], "oil_bbl / day") + output_commodity_quantities['diesel'] = Q_(bioWasteDict[commodity][process][6], "oil_bbl / day") + output_commodity_quantities['naphtha'] = Q_(bioWasteDict[commodity][process][7], "oil_bbl / day") + output_commodity_quantities['aromatics'] = Q_(0.000, "oil_bbl / day") + + else: + + logger.debug( + "the process {} for commodity {} is not in the biowaste yield dictionary {}".format(process, commodity, + bioWasteDict[ + commodity])) + logger.error( + "the commodity {} has no process {} in the AFPAT biowaste yield dictionary".format(commodity, process)) + raise Exception( + "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, + process)) + + # DO THE FOSSIL RESOURCES + fossil_keys = list(fossilResources.keys()) + + for key in fossil_keys: + + if commodity.find(key) > -1: + + if process in fossilResources[key]: + + input_commodity_quantities = Q_(500e3, "oil_bbl / day") + + output_commodity_quantities['total_fuel'] = Q_(fossilResources[key][process][4], "oil_bbl / day") + output_commodity_quantities['jet'] = Q_(fossilResources[key][process][5], "oil_bbl / day") + output_commodity_quantities['diesel'] = Q_(fossilResources[key][process][6], "oil_bbl / day") + output_commodity_quantities['naphtha'] = Q_(fossilResources[key][process][7], "oil_bbl / day") + output_commodity_quantities['aromatics'] = Q_(0.000, "oil_bbl / day") + + else: + + logger.error( + "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, + process)) + raise Exception( + "the commodity {} has no process {} in the AFPAT fossilResources yield dictionary".format(commodity, + process)) + + if input_commodity_quantities == 0 or output_commodity_quantities == {}: + logger.error( + "the commodity {} and process {} is not in the AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( + commodity, process)) + raise Exception( + "the commodity {} and process {} is not in the AFPAT agricultural, biowaste, or fossil fuel yield dictionaries".format( + commodity, process)) + + return input_commodity_quantities, output_commodity_quantities + + +# ============================================================================= +def get_RMP_commodity_list(the_scenario, logger): + import sqlite3 + + main_db = the_scenario.main_db + RMP_commodity_list = [] + + with sqlite3.connect(main_db) as db_con: + db_cur = db_con.cursor() + + sql = "select distinct commodity from raw_material_producers;" + db_cur.execute(sql) + + for record in db_cur: + RMP_commodity_list.append(record[0]) + + return RMP_commodity_list + + +# ============================================================================== +def get_route_type(commodity_name, rmp_commodity_list): + route_type = "" + + if commodity_name in rmp_commodity_list: + + route_type = "RMP_Processor" + + else: + + route_type = "Processor_Destination" + + return route_type + + +# ============================================================================== + +def get_commodity_simple_name(commodity_name): + simple_commodity_name = "" + + if commodity_name.find("diesel") >= 0: + + simple_commodity_name = "diesel" + + elif commodity_name.find("jet") >= 0: + + simple_commodity_name = "jet" + + else: + + simple_commodity_name = commodity_name # a raw material producer material that doesn't have a simple name + + return simple_commodity_name + + +# ============================================================================== +def post_optimization(the_scenario, task_id, logger): + from ftot_pulp import parse_optimal_solution_db + + logger.info("START: post_optimization for {} task".format(task_id)) + start_time = datetime.datetime.now() + + # Parse the Problem for the Optimal Solution + parsed_optimal_solution = parse_optimal_solution_db(the_scenario, logger) + + logger.info("FINISH: post_optimization: Runtime (HMS): \t{}".format(get_total_runtime_string(start_time))) + +# =================================================================================================== + + +def debug_write_solution(the_scenario, prob, logfile, logger): + logger.info("starting debug_write_solution") + + with open(os.path.join(the_scenario.scenario_run_directory, "logs", logfile), 'w') as solution_file: + + for v in prob.variables(): + + if v.varValue > 0.0: + solution_file.write("{} -> {:,.2f}\n".format(v.name, v.varValue)) + logger.info("finished: debug_write_solution") diff --git a/program/ftot_supporting_gis.py b/program/ftot_supporting_gis.py index 5f4351f..4405333 100644 --- a/program/ftot_supporting_gis.py +++ b/program/ftot_supporting_gis.py @@ -1,606 +1,715 @@ -# ----------------------------------------------------------------------------- -# Name: ftot_suppporting_gis -# -# Purpose: -# -# ----------------------------------------------------------------------------- - -import os -import math -import itertools -import arcpy -import sqlite3 -LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') -from ftot import Q_ - -THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 - - -# =================================================================================================== - - -def get_commodity_vehicle_attributes_dict(the_scenario, logger): - - with sqlite3.connect(the_scenario.main_db) as main_db_con: - - # query commodities table - commodities_dict = {} # key off ID - commodities = main_db_con.execute("select * from commodities where commodity_name <> 'multicommodity';") - commodities = commodities.fetchall() - for row in commodities: - commodity_id = str(row[0]) - commodity_name = row[1] - commodities_dict[commodity_id] = commodity_name - - # query commodity modes table - commodity_mode_dict = {} # key off phase, then commodity name (not ID!) - commodity_mode = main_db_con.execute("select * from commodity_mode;") - commodity_mode = commodity_mode.fetchall() - for row in commodity_mode: - mode = row[0] - commodity_id = row[1] - phase = row[2] - vehicle_label = row[3] - allowed_yn = row[4] - if allowed_yn == 'N': - continue # skip row if not permitted - - # use commodities dictionary to determine commodity name - commodity_name = commodities_dict[commodity_id] - - # populate commodity modes dictionary - if phase not in commodity_mode_dict: - # add new phase to dictionary and start commodity and mode dictionary - commodity_mode_dict[phase] = {commodity_name: {mode: vehicle_label}} - elif commodity_name not in commodity_mode_dict[phase]: - # add new commodity to dictionary and start mode dictionary - commodity_mode_dict[phase][commodity_name] = {mode: vehicle_label} - else: - # add new mode to dictionary - commodity_mode_dict[phase][commodity_name][mode] = vehicle_label - - # query vehicle types table - vehicle_types_dict = {} # key off mode - vehs = main_db_con.execute("select * from vehicle_types;") - vehs = vehs.fetchall() - for row in vehs: - mode = row[0] - vehicle_label = row[1] - property_name = row[2] - property_value = Q_(row[3]) - if mode not in vehicle_types_dict: - # add new mode to dictionary and start vehicle and property dictionary - vehicle_types_dict[mode] = {vehicle_label: {property_name: property_value}} - elif vehicle_label not in vehicle_types_dict[mode]: - # add new vehicle to dictionary and start property dictionary - vehicle_types_dict[mode][vehicle_label] = {property_name: property_value} - else: - # add to existing dictionary entry - vehicle_types_dict[mode][vehicle_label][property_name] = property_value - - # create commodity/vehicle attribute dictionary - logger.debug("----- commodity/vehicle attribute table -----") - - solid_to_liquid_factor = Q_('2.87 ton/kgal').to('{}/{}'.format(the_scenario.default_units_solid_phase, the_scenario.default_units_liquid_phase)) # based on representative commodity density - - attribute_dict = {} # key off commodity name - for phase in commodity_mode_dict: - for commodity_name in commodity_mode_dict[phase]: - for mode in commodity_mode_dict[phase][commodity_name]: - - # Get vehicle assigned to commodity on this mode - vehicle_label = commodity_mode_dict[phase][commodity_name][mode] - - if commodity_name not in attribute_dict: - # Create dictionary entry for commodity - attribute_dict[commodity_name] = {mode: {}} - else: - # Create dictionary entry for commodity's mode - attribute_dict[commodity_name][mode] = {} - - # Set attributes based on mode, vehicle label, and commodity phase - # ROAD - if mode == 'road': - if vehicle_label == 'Default': - # use default attributes for trucks - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = the_scenario.truck_load_liquid - else: - attribute_dict[commodity_name][mode]['Load'] = the_scenario.truck_load_solid - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.truckFuelEfficiency - attribute_dict[commodity_name][mode]['CO2urbanUnrestricted'] = the_scenario.CO2urbanUnrestricted - attribute_dict[commodity_name][mode]['CO2urbanRestricted'] = the_scenario.CO2urbanRestricted - attribute_dict[commodity_name][mode]['CO2ruralUnrestricted'] = the_scenario.CO2ruralUnrestricted - attribute_dict[commodity_name][mode]['CO2ruralRestricted'] = the_scenario.CO2ruralRestricted - - elif vehicle_label != 'NA': - # use user-specified vehicle attributes, or if missing, the default value - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Truck_Load_Liquid'] - else: - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Truck_Load_Solid'] - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Truck_Fuel_Efficiency'] - attribute_dict[commodity_name][mode]['CO2urbanUnrestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Urban_Unrestricted'] - attribute_dict[commodity_name][mode]['CO2urbanRestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Urban_Restricted'] - attribute_dict[commodity_name][mode]['CO2ruralUnrestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Rural_Unrestricted'] - attribute_dict[commodity_name][mode]['CO2ruralRestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Rural_Restricted'] - - # RAIL - elif mode == 'rail': - if vehicle_label == 'Default': - # use default attributes for railcars - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = the_scenario.railcar_load_liquid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * the_scenario.railroadCO2Emissions - else: - attribute_dict[commodity_name][mode]['Load'] = the_scenario.railcar_load_solid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.railroadCO2Emissions - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.railFuelEfficiency - - elif vehicle_label != 'NA': - # use user-specified vehicle attributes, or if missing, the default value - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Railcar_Load_Liquid'] - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * vehicle_types_dict[mode][vehicle_label]['Railroad_CO2_Emissions'] - else: - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Railcar_Load_Solid'] - attribute_dict[commodity_name][mode]['CO2_Emissions'] = vehicle_types_dict[mode][vehicle_label]['Railroad_CO2_Emissions'] - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Rail_Fuel_Efficiency'] - - # WATER - elif mode == 'water': - if vehicle_label == 'Default': - # use default attributes barges - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = the_scenario.barge_load_liquid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * the_scenario.bargeCO2Emissions - else: - attribute_dict[commodity_name][mode]['Load'] = the_scenario.barge_load_solid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.bargeCO2Emissions - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.bargeFuelEfficiency - - elif vehicle_label != 'NA': - # use user-specified vehicle attributes, or if missing, the default value - if phase == 'liquid': - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Barge_Load_Liquid'] - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * vehicle_types_dict[mode][vehicle_label]['Barge_CO2_Emissions'] - else: - attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Barge_Load_Solid'] - attribute_dict[commodity_name][mode]['CO2_Emissions'] = vehicle_types_dict[mode][vehicle_label]['Barge_CO2_Emissions'] - - attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Barge_Fuel_Efficiency'] - - elif mode == 'pipeline_crude_trf_rts': - attribute_dict[commodity_name][mode]['Load'] = the_scenario.pipeline_crude_load_liquid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * the_scenario.pipelineCO2Emissions - - elif mode == 'pipeline_prod_trf_rts': - attribute_dict[commodity_name][mode]['Load'] = the_scenario.pipeline_prod_load_liquid - attribute_dict[commodity_name][mode]['CO2_Emissions'] = solid_to_liquid_factor * the_scenario.pipelineCO2Emissions - - for attr in attribute_dict[commodity_name][mode].keys(): - attr_value = attribute_dict[commodity_name][mode][attr] - logger.debug("Commodity: {}, Mode: {}, Attribute: {}, Value: {}".format(commodity_name, mode, attr, attr_value)) - - return attribute_dict # Keyed off of commodity name, then mode, then vehicle attribute - -# ============================================================================= - - -STATES_DATA = [ -['01', 'AL', 'ALABAMA'], -['02', 'AK', 'ALASKA'], -['04', 'AZ', 'ARIZONA'], -['05', 'AR', 'ARKANSAS'], -['06', 'CA', 'CALIFORNIA'], -['08', 'CO', 'COLORADO'], -['09', 'CT', 'CONNECTICUT'], -['10', 'DE', 'DELAWARE'], -['11', 'DC', 'DISTRICT_OF_COLUMBIA'], -['12', 'FL', 'FLORIDA'], -['13', 'GA', 'GEORGIA'], -['15', 'HI', 'HAWAII'], -['16', 'ID', 'IDAHO'], -['17', 'IL', 'ILLINOIS'], -['18', 'IN', 'INDIANA'], -['19', 'IA', 'IOWA'], -['20', 'KS', 'KANSAS'], -['21', 'KY', 'KENTUCKY'], -['22', 'LA', 'LOUISIANA'], -['23', 'ME', 'MAINE'], -['24', 'MD', 'MARYLAND'], -['25', 'MA', 'MASSACHUSETTS'], -['26', 'MI', 'MICHIGAN'], -['27', 'MN', 'MINNESOTA'], -['28', 'MS', 'MISSISSIPPI'], -['29', 'MO', 'MISSOURI'], -['30', 'MT', 'MONTANA'], -['31', 'NE', 'NEBRASKA'], -['32', 'NV', 'NEVADA'], -['33', 'NH', 'NEW_HAMPSHIRE'], -['34', 'NJ', 'NEW_JERSEY'], -['35', 'NM', 'NEW_MEXICO'], -['36', 'NY', 'NEW_YORK'], -['37', 'NC', 'NORTH_CAROLINA'], -['38', 'ND', 'NORTH_DAKOTA'], -['39', 'OH', 'OHIO'], -['40', 'OK', 'OKLAHOMA'], -['41', 'OR', 'OREGON'], -['42', 'PA', 'PENNSYLVANIA'], -['72', 'PR', 'PUERTO_RICO'], -['44', 'RI', 'RHODE_ISLAND'], -['45', 'SC', 'SOUTH_CAROLINA'], -['46', 'SD', 'SOUTH_DAKOTA'], -['47', 'TN', 'TENNESSEE'], -['48', 'TX', 'TEXAS'], -['49', 'UT', 'UTAH'], -['50', 'VT', 'VERMONT'], -['51', 'VA', 'VIRGINIA'], -['53', 'WA', 'WASHINGTON'], -['54', 'WV', 'WEST_VIRGINIA'], -['55', 'WI', 'WISCONSIN'], -['56', 'WY', 'WYOMING'] -] - -# ======================================================================================================================= - - -def get_state_abb_from_state_fips(input_fips): - - return_value = 'XX' - - for state_data in STATES_DATA: - if state_data[0] == input_fips: - return_value = state_data[1] - - return return_value - -# ===================================================================================================================== - - -def assign_pipeline_costs(the_scenario, logger, include_pipeline): - scenario_gdb = os.path.join(the_scenario.scenario_run_directory, "main.gdb") - # calc the cost for pipeline - # --------------------- - - if include_pipeline: - logger.info("starting calculate cost for PIPELINE") - # base rate is cents per barrel - - # calculate for all - arcpy.MakeFeatureLayer_management (os.path.join(scenario_gdb, "pipeline"), "pipeline_lyr") - arcpy.SelectLayerByAttribute_management(in_layer_or_view="pipeline_lyr", selection_type="NEW_SELECTION", where_clause="Artificial = 0") - arcpy.CalculateField_management("pipeline_lyr", field="FROM_TO_ROUTING_COST", expression="(((!base_rate! / 100) / 42.0) * 1000.0)", expression_type="PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", field="FROM_TO_DOLLAR_COST", expression="(((!base_rate! / 100) / 42.0) * 1000.0)", expression_type="PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") - - # if base_rate is null, select those links that are not artifiical and set to 0 manually. - arcpy.SelectLayerByAttribute_management(in_layer_or_view="pipeline_lyr", selection_type="NEW_SELECTION", where_clause="Artificial = 0 and base_rate is null") - arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_ROUTING_COST", 0, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_DOLLAR_COST", 0, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") - no_baserate_count = int(arcpy.GetCount_management("pipeline_lyr").getOutput(0)) - if no_baserate_count > 0: - logger.warn("pipeline network contains {} routes with no base_rate. Flow cost will be set to 0".format(no_baserate_count)) - - else: - logger.info("starting calculate cost for PIPELINE = -1") - arcpy.MakeFeatureLayer_management (os.path.join(scenario_gdb, "pipeline"), "pipeline_lyr") - arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_ROUTING_COST", -1, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_DOLLAR_COST", -1, "PYTHON_9.3") - arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") - -# ====================================================================================================================== - - -def set_intermodal_links(the_scenario, logger): - scenario_gdb = the_scenario.main_gdb - # set the artificial link field to 2 so that it - # does not have flow restrictions applied to it. - - # assumes that the only artificial links in the network - # are connecting intermodal facilities. - - logger.info("start: set_intermodal_links") - - for mode in ["road", "rail", "pipeline_prod_trf_rts", "pipeline_crude_trf_rts", "water"]: - mode_layer = "{}_lyr".format(mode) - arcpy.MakeFeatureLayer_management(os.path.join(scenario_gdb, mode), mode_layer) - arcpy.SelectLayerByAttribute_management(in_layer_or_view=mode_layer, - selection_type="NEW_SELECTION", - where_clause="Artificial = 1") - arcpy.CalculateField_management(mode_layer, "ARTIFICIAL", 2, "PYTHON_9.3") - -# ====================================================================================================================== - - -class LicenseError(Exception): - pass - -# ====================================================================================================================== - - -def load_afpat_data_to_memory(fullPathToTable, logger): - - """load afpat data to memory""" - - logger.debug("START: load_afpat_data_to_memory") - - crop_yield_dict = {} # keyed off crop name --> crop yield (e.g. kg/ha/yr) - fuel_yield_dict = {} # keyed off crop name + processing type --> fuel yields (e.g. jet, diesel, total biomass) - - afpatAsList = [] - - # iterate through the afpat table and save as a list - # -------------------------------------------- - with arcpy.da.SearchCursor(fullPathToTable, "*") as cursor: - - for row in cursor: - - afpatAsList.append(row) - - # now identify the relevant indexs for the values we want to extract - - # land specific information - # ----------------------------------------- - countryIndex = afpatAsList[34].index('Country') - landAreaIndex = afpatAsList[34].index('Total Land Area Used') - - - # feedstock names - # ----------------------------------- - Feedstock_Source_Index = afpatAsList[34].index('Feedstock Source') - feedstock_type = afpatAsList[34].index('Feedstock Type') - feedstock_source_category = afpatAsList[34].index('Source Category') - - # crop yield index - #---------------------------- - oilIndex = afpatAsList[34].index('Oil Yield') - noncellulosicIndex = afpatAsList[34].index('Non-Cellulosic Sugar Yield') - lignoIndex = afpatAsList[34].index('Lignocellulosic Yield') - - # process index - # ------------------------------------------- - primaryProcTypeIndex = afpatAsList[34].index('Primary Processing Type') - secondaryProcTypeIndex = afpatAsList[34].index('Secondary Processing Type') - tertiaryProcTypeIndex = afpatAsList[34].index('Tertiary Processing Type') - capital_costs = afpatAsList[34].index('Capital Costs') - - # fuel yeilds index - # ------------------------------------------- - jetFuelIndex = afpatAsList[34].index('Jet Fuel ') - dieselFuelIndex = afpatAsList[34].index('Diesel fuel') - naphthaIndex = afpatAsList[34].index('Naphtha') - aromaticsIndex = afpatAsList[34].index('Aromatics') - total_fuel = afpatAsList[34].index('Total fuel (Excluding propane, LPG and heavy oil)') - conversion_efficiency = afpatAsList[34].index('Conversion Eff') - total_daily_biomass = afpatAsList[34].index('Total Daily Biomass') - - - table2cIndex = 0 - table2dIndex = 0 - - from ftot_supporting import get_cleaned_process_name - from ftot_supporting import create_full_crop_name - - for r in afpatAsList: - - # move biowaste to another table - if r[2] == "Table 2c:": - - table2cIndex = afpatAsList.index(r) - - # move fossil resources to another table - elif r[2] == "Table 2d:": - - table2dIndex = afpatAsList.index(r) - - # add commodity and process information to the crop_yield_dict and fuel_yield_dict - if r[countryIndex] == "U.S." and r[Feedstock_Source_Index] != "": - - # concatinate crop name - feedstock_name = create_full_crop_name(r[feedstock_type], r[feedstock_source_category],r[Feedstock_Source_Index]) - - - - # determine which crop yield field we want depending on the feedstock_type - if r[feedstock_type] == "Oils": - - preprocYield = float(r[oilIndex]) - - elif r[feedstock_type] == "Non-Cellulosic Sugars": - - preprocYield = float(r[noncellulosicIndex]) - - elif r[feedstock_type] == "Lignocellulosic Biomass and Cellulosic Sugars": - - preprocYield = float(r[lignoIndex]) - - else: - logger.error("Error in feedstock type specification from AFPAT") - raise Exception("Error in feedstock type specification from AFPAT") - - - # preprocYield is independent of processor type, so its okay - # if this gets written over by the next instance of the commodity - crop_yield_dict[feedstock_name] = preprocYield - - # Fuel Yield Dictionary - # --------------------------------- - # each commodity gets its own dictionary, keyed on the 3 process types; value is a list of the form - # [land area, jet, diesel, naphtha, aromatics, yield]; I'm adding total fuel and - # feedstock type to the end of this list; maybe units also? - - process_name = get_cleaned_process_name(r[primaryProcTypeIndex], r[secondaryProcTypeIndex],r[tertiaryProcTypeIndex]) - - if feedstock_name not in fuel_yield_dict: - - fuel_yield_dict[feedstock_name] = {process_name: [float(r[landAreaIndex]), - float(r[jetFuelIndex]), float(r[dieselFuelIndex]), - float(r[naphthaIndex]), float(r[aromaticsIndex]), - preprocYield, float(r[total_fuel]), - float(r[conversion_efficiency]), float(r[total_daily_biomass])]} - else: - - fuel_yield_dict[feedstock_name][process_name] = [float(r[landAreaIndex]), - float(r[jetFuelIndex]), float(r[dieselFuelIndex]), - float(r[naphthaIndex]), float(r[aromaticsIndex]), - preprocYield, float(r[total_fuel]), - float(r[conversion_efficiency]), float(r[total_daily_biomass])] - - table2c = list(itertools.islice(afpatAsList, table2cIndex + 2, table2dIndex)) - table2d = list(itertools.islice(afpatAsList, table2dIndex + 2, len(afpatAsList))) - - bioWasteDict = {} - fossilResources = {} - - resourceIndex = table2c[0].index("Resource") - procTypeIndex = table2c[0].index("Processing Type") - percentIndex = table2c[0].index("Percent of Resource") - resourceKgIndex = table2c[1].index("kg/yr") - facilitiesIndex = table2c[0].index("# FTx or HEFA Facilities") - capCostsIndex = table2c[0].index("Capital Costs") - totFuelIndex = table2c[0].index("Total fuel") - jetFuelIndex = table2c[0].index("Jet fuel") - dieselFuelIndex = table2c[0].index("Diesel fuel") - naphthaIndex = table2c[0].index("naphtha") - - numFtxIndex = table2d[0].index("# FTx Facilities") - ccsIndex = table2d[0].index("CCS Required") - eorIndex = table2d[0].index("Percent of EOR Capacity") - tabDcapCost = table2d[0].index("Capital Costs") - - secondaryProcType = "N/A" - tertiaryProcType = "N/A" - - - del table2c[0] - del table2d[0] - - for t in table2c: - - if t[2] != "": - - resourceName = t[resourceIndex].replace("-", "_").replace(" ", "_") - processType = (t[procTypeIndex].replace("-", "_").replace(" ", "_"), secondaryProcType, tertiaryProcType) - - if resourceName not in bioWasteDict: - - bioWasteDict[resourceName] = {processType: [float(t[percentIndex]), - float(t[resourceKgIndex]), - int(math.ceil(float( t[facilitiesIndex]))), - float(t[capCostsIndex]), - int(math.ceil(float( t[totFuelIndex]))), - int(math.ceil(float( t[jetFuelIndex]))), - int(math.ceil(float( t[dieselFuelIndex]))), - int(math.ceil(float( t[naphthaIndex])))]} - else: - - bioWasteDict[resourceName][processType] = [float(t[percentIndex]), - float(t[resourceKgIndex]), - int(math.ceil(float( t[facilitiesIndex]))), - float(t[capCostsIndex]), - int(math.ceil(float( t[totFuelIndex]))), - int(math.ceil(float( t[jetFuelIndex]))), - int(math.ceil(float( t[dieselFuelIndex]))), - int(math.ceil(float( t[naphthaIndex])))] - for t in table2d: - if t[2] != "" and t[3] != "": - - resourceName = t[resourceIndex].replace("-", "_").replace(" ", "_") - processType = (t[procTypeIndex].replace("-", "_").replace(" ", "_"), secondaryProcType, tertiaryProcType) - - if resourceName not in fossilResources: - - fossilResources[resourceName] = {processType: [int(t[numFtxIndex]), - float(t[tabDcapCost]), - float(t[ccsIndex]), - float(t[eorIndex]), - int(math.ceil(float( t[totFuelIndex]))), - int(math.ceil(float( t[jetFuelIndex]))), - int(math.ceil(float( t[dieselFuelIndex]))), - int(math.ceil(float( t[naphthaIndex])))]} - - - else: - - fossilResources[resourceName][processType] = [int(t[numFtxIndex]), - float(t[tabDcapCost]), - float(t[ccsIndex]), - float(t[eorIndex]), - int(math.ceil(float( t[totFuelIndex]))), - int(math.ceil(float( t[jetFuelIndex]))), - int(math.ceil(float( t[dieselFuelIndex]))), - int(math.ceil(float( t[naphthaIndex])))] - - logger.debug("FINISH: load_afpat_data_to_memory") - - return fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources -# ============================================================================== - - -def persist_AFPAT_tables(the_scenario, logger): - # pickle the AFPAT tables so it can be read in later without ArcPy - - scenario_gdb = the_scenario.main_gdb - - afpat_raw_table = os.path.join(scenario_gdb, "afpat_raw") - - fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources = load_afpat_data_to_memory(afpat_raw_table, logger) - - afpat_tables = [fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources] - -# ============================================================================== - -# ********************************************************************** -# File name: Zip.py -# Description: -# Zips the contents of a folder, file geodatabase or ArcInfo workspace -# containing coverages into a zip file. -# Arguments: -# 0 - Input workspace -# 1 - Output zip file. A .zip extension should be added to the file name -# -# Created by: ESRI -# ********************************************************************** - -# Import modules and create the geoprocessor -import os - - -# Function for zipping files -def zipgdb(path, zip_file): - isdir = os.path.isdir - - # Check the contents of the workspace, if it the current - # item is a directory, gets its contents and write them to - # the zip file, otherwise write the current file item to the - # zip file - for each in os.listdir(path): - fullname = path + "/" + each - if not isdir(fullname): - # If the workspace is a file geodatabase, avoid writing out lock - # files as they are unnecessary - if not each.endswith('.lock'): - # Write out the file and give it a relative archive path - try: zip_file.write(fullname, each) - except IOError: None # Ignore any errors in writing file - else: - # Branch for sub-directories - for eachfile in os.listdir(fullname): - if not isdir(eachfile): - if not each.endswith('.lock'): - gp.AddMessage("Adding " + eachfile + " ...") - # Write out the file and give it a relative archive path - try: zip_file.write(fullname + "/" + eachfile, os.path.basename(fullname) + "/" + eachfile) - except IOError: None # Ignore any errors in writing file - - +# ----------------------------------------------------------------------------- +# Name: ftot_suppporting_gis +# +# Purpose: +# +# ----------------------------------------------------------------------------- + +import os +import math +import itertools +import arcpy +import sqlite3 +LCC_PROJ = arcpy.SpatialReference('USA Contiguous Lambert Conformal Conic') +from ftot import Q_ + +THOUSAND_GALLONS_PER_THOUSAND_BARRELS = 42 + + +# =================================================================================================== + +def make_emission_factors_dict(the_scenario, logger): + + # check for emission factors file + ftot_program_directory = os.path.dirname(os.path.realpath(__file__)) + emission_factors_path = os.path.join(ftot_program_directory, "lib", "detailed_emission_factors.csv") + if not os.path.exists(emission_factors_path): + logger.warning("warning: cannot find detailed_emission_factors file: {}".format(emission_factors_path)) + return {} # return empty dict + + # query vehicle labels for validation + available_vehicles = ['Default'] + with sqlite3.connect(the_scenario.main_db) as main_db_con: + db_cur = main_db_con.cursor() + all_vehicles = main_db_con.execute("select vehicle_label from vehicle_types;") + all_vehicles = all_vehicles.fetchall() + for row in all_vehicles: + available_vehicles.append(row[0]) + available_vehicles = set(available_vehicles) + + # initialize emission factors dict and read through detailed_emission_factors CSV + factors_dict = {} + with open(emission_factors_path, 'r') as ef: + line_num = 1 + for line in ef: + if line_num == 1: + pass # do nothing + else: + flds = line.rstrip('\n').split(',') + vehicle_label = flds[0] + mode = flds[1].lower() + road_type = flds[2] + pollutant = flds[3].lower() + factor = flds[4] + + # Check vehicle label + # Note: We're strict with capitalization here since vehicle_types processing is also strict + if vehicle_label not in available_vehicles: + logger.warning("Vehicle: {} in detailed emissions files is not recognized.".format(vehicle_label)) + + # Check mode + assert mode in ['road', 'water', 'rail'], "Mode: {} is not supported. Please specify road, water, or rail.".format(mode) + + # Check road type + if mode == 'road': + allowed_road_types = ['Urban_Unrestricted', 'Urban_Restricted', 'Rural_Unrestricted', 'Rural_Restricted'] + assert road_type in allowed_road_types, "Road type: {} is not recognized. Road type must be one of {}.".format(road_type, allowed_road_types) + else: + assert road_type == 'NA', "Road type must be 'NA' for water and rail modes." + + assert pollutant in ['co','co2e','ch4','n2o','nox','pm10','pm2.5','voc'],\ + "Pollutant: {} is not recognized. Refer to the documentation for allowed pollutants.".format(pollutant) + + # convert units + # Pint throws an exception if units are invalid + if mode == 'road': + factor = Q_(factor).to('g/mi') + else: + factor = Q_(factor).to('g/{}/mi'.format(the_scenario.default_units_solid_phase)) + + # populate dictionary + if mode not in factors_dict: + # create entry for new mode type + factors_dict[mode] = {} + if mode == 'road': + # store emission factors for road + if vehicle_label not in factors_dict[mode]: + factors_dict[mode][vehicle_label] = {pollutant: {road_type: factor}} + elif pollutant not in factors_dict[mode][vehicle_label]: + factors_dict[mode][vehicle_label][pollutant] = {road_type: factor} + else: + if road_type in factors_dict[mode][vehicle_label][pollutant].keys(): + logger.warning('Road type: {} for pollutant: {} and vehicle: {} already exists. Overwriting with value: {}'.\ + format(road_type, pollutant, vehicle_label, factor)) + factors_dict[mode][vehicle_label][pollutant][road_type] = factor + else: + # store emission factors for non-road + if vehicle_label not in factors_dict[mode]: + factors_dict[mode][vehicle_label] = {pollutant: factor} + else: + if pollutant in factors_dict[mode][vehicle_label].keys(): + logger.warning('Pollutant: {} already exists for vehicle: {}. Overwriting with value: {}'.format(pollutant, vehicle_label, factor)) + factors_dict[mode][vehicle_label][pollutant] = factor + + line_num += 1 + + return factors_dict + + +# =================================================================================================== + +def get_commodity_vehicle_attributes_dict(the_scenario, logger, EmissionsWarning=False): + + with sqlite3.connect(the_scenario.main_db) as main_db_con: + + # query commodities table + commodities_dict = {} # key off ID + commodities = main_db_con.execute("select * from commodities where commodity_name <> 'multicommodity';") + commodities = commodities.fetchall() + for row in commodities: + commodity_id = str(row[0]) + commodity_name = row[1] + commodities_dict[commodity_id] = commodity_name + + # query commodity modes table + commodity_mode_dict = {} # key off phase, then commodity name (not ID!) + commodity_mode = main_db_con.execute("select * from commodity_mode;") + commodity_mode = commodity_mode.fetchall() + for row in commodity_mode: + mode = row[0] + commodity_id = row[1] + phase = row[2] + vehicle_label = row[3] + allowed_yn = row[4] + if allowed_yn == 'N': + continue # skip row if not permitted + + # use commodities dictionary to determine commodity name + commodity_name = commodities_dict[commodity_id] + + # populate commodity modes dictionary + if phase not in commodity_mode_dict: + # add new phase to dictionary and start commodity and mode dictionary + commodity_mode_dict[phase] = {commodity_name: {mode: vehicle_label}} + elif commodity_name not in commodity_mode_dict[phase]: + # add new commodity to dictionary and start mode dictionary + commodity_mode_dict[phase][commodity_name] = {mode: vehicle_label} + else: + # add new mode to dictionary + commodity_mode_dict[phase][commodity_name][mode] = vehicle_label + + # query vehicle types table + vehicle_types_dict = {} # key off mode + vehs = main_db_con.execute("select * from vehicle_types;") + vehs = vehs.fetchall() + for row in vehs: + mode = row[0] + vehicle_label = row[1] + property_name = row[2] + property_value = Q_(row[3]) + if mode not in vehicle_types_dict: + # add new mode to dictionary and start vehicle and property dictionary + vehicle_types_dict[mode] = {vehicle_label: {property_name: property_value}} + elif vehicle_label not in vehicle_types_dict[mode]: + # add new vehicle to dictionary and start property dictionary + vehicle_types_dict[mode][vehicle_label] = {property_name: property_value} + else: + # add to existing dictionary entry + vehicle_types_dict[mode][vehicle_label][property_name] = property_value + + # load detailed emission factors + factors_dict = make_emission_factors_dict(the_scenario, logger) + + # create commodity/vehicle attribute dictionary + logger.debug("----- commodity/vehicle attribute table -----") + + attribute_dict = {} # key off commodity name + for phase in commodity_mode_dict: + for commodity_name in commodity_mode_dict[phase]: + for mode in commodity_mode_dict[phase][commodity_name]: + + # Get vehicle assigned to commodity on this mode + vehicle_label = commodity_mode_dict[phase][commodity_name][mode] + + if commodity_name not in attribute_dict: + # Create dictionary entry for commodity + attribute_dict[commodity_name] = {mode: {}} + else: + # Create dictionary entry for commodity's mode + attribute_dict[commodity_name][mode] = {} + + # Set attributes based on mode, vehicle label, and commodity phase + # ROAD + if mode == 'road': + if vehicle_label == 'Default': + # use default attributes for trucks + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = the_scenario.truck_load_liquid + else: + attribute_dict[commodity_name][mode]['Load'] = the_scenario.truck_load_solid + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.truckFuelEfficiency + attribute_dict[commodity_name][mode]['CO2urbanUnrestricted'] = the_scenario.CO2urbanUnrestricted + attribute_dict[commodity_name][mode]['CO2urbanRestricted'] = the_scenario.CO2urbanRestricted + attribute_dict[commodity_name][mode]['CO2ruralUnrestricted'] = the_scenario.CO2ruralUnrestricted + attribute_dict[commodity_name][mode]['CO2ruralRestricted'] = the_scenario.CO2ruralRestricted + + elif vehicle_label != 'NA': + # use user-specified vehicle attributes, or if missing, the default value + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Truck_Load_Liquid'] + else: + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Truck_Load_Solid'] + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Truck_Fuel_Efficiency'] + attribute_dict[commodity_name][mode]['CO2urbanUnrestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Urban_Unrestricted'] + attribute_dict[commodity_name][mode]['CO2urbanRestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Urban_Restricted'] + attribute_dict[commodity_name][mode]['CO2ruralUnrestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Rural_Unrestricted'] + attribute_dict[commodity_name][mode]['CO2ruralRestricted'] = vehicle_types_dict[mode][vehicle_label]['Atmos_CO2_Rural_Restricted'] + + # RAIL + elif mode == 'rail': + if vehicle_label == 'Default': + # use default attributes for railcars + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = the_scenario.railcar_load_liquid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * the_scenario.railroadCO2Emissions + else: + attribute_dict[commodity_name][mode]['Load'] = the_scenario.railcar_load_solid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.railroadCO2Emissions + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.railFuelEfficiency + + elif vehicle_label != 'NA': + # use user-specified vehicle attributes, or if missing, the default value + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Railcar_Load_Liquid'] + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * vehicle_types_dict[mode][vehicle_label]['Railroad_CO2_Emissions'] + else: + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Railcar_Load_Solid'] + attribute_dict[commodity_name][mode]['CO2_Emissions'] = vehicle_types_dict[mode][vehicle_label]['Railroad_CO2_Emissions'] + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Rail_Fuel_Efficiency'] + + # WATER + elif mode == 'water': + if vehicle_label == 'Default': + # use default attributes barges + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = the_scenario.barge_load_liquid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * the_scenario.bargeCO2Emissions + else: + attribute_dict[commodity_name][mode]['Load'] = the_scenario.barge_load_solid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.bargeCO2Emissions + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = the_scenario.bargeFuelEfficiency + + elif vehicle_label != 'NA': + # use user-specified vehicle attributes, or if missing, the default value + if phase == 'liquid': + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Barge_Load_Liquid'] + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * vehicle_types_dict[mode][vehicle_label]['Barge_CO2_Emissions'] + else: + attribute_dict[commodity_name][mode]['Load'] = vehicle_types_dict[mode][vehicle_label]['Barge_Load_Solid'] + attribute_dict[commodity_name][mode]['CO2_Emissions'] = vehicle_types_dict[mode][vehicle_label]['Barge_CO2_Emissions'] + + attribute_dict[commodity_name][mode]['Fuel_Efficiency'] = vehicle_types_dict[mode][vehicle_label]['Barge_Fuel_Efficiency'] + + # PIPELINE + elif mode == 'pipeline_crude_trf_rts': + attribute_dict[commodity_name][mode]['Load'] = the_scenario.pipeline_crude_load_liquid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * the_scenario.pipelineCO2Emissions + + elif mode == 'pipeline_prod_trf_rts': + attribute_dict[commodity_name][mode]['Load'] = the_scenario.pipeline_prod_load_liquid + attribute_dict[commodity_name][mode]['CO2_Emissions'] = the_scenario.densityFactor * the_scenario.pipelineCO2Emissions + + # DETAILED EMISSION FACTORS + # Include detailed emission factors + if mode in factors_dict and vehicle_label in factors_dict[mode]: + # loop through emission factors + for pollutant in factors_dict[mode][vehicle_label]: + if mode in ['rail','water'] and phase == 'liquid': + attribute_dict[commodity_name][mode][pollutant] = the_scenario.densityFactor * factors_dict[mode][vehicle_label][pollutant] + else: + attribute_dict[commodity_name][mode][pollutant] = factors_dict[mode][vehicle_label][pollutant] + + # Code block below checks if user assigns custom vehicle without detailed emission factors --> + if mode in vehicle_types_dict and vehicle_label in vehicle_types_dict[mode]: + # user used a custom vehicle. check if wants detailed emissions reporting. + if the_scenario.detailed_emissions and EmissionsWarning: + # warn if don't have matching emission factors + if mode not in factors_dict or vehicle_label not in factors_dict[mode]: + logger.warning("Detailed emission factors are not specified for vehicle: {} for mode: {}. Excluding this vehicle from the emissions report.".format(vehicle_label, mode)) + + for attr in attribute_dict[commodity_name][mode].keys(): + attr_value = attribute_dict[commodity_name][mode][attr] + logger.debug("Commodity: {}, Mode: {}, Attribute: {}, Value: {}".format(commodity_name, mode, attr, attr_value)) + + return attribute_dict # Keyed off of commodity name, then mode, then vehicle attribute + +# ============================================================================= + + +STATES_DATA = [ +['01', 'AL', 'ALABAMA'], +['02', 'AK', 'ALASKA'], +['04', 'AZ', 'ARIZONA'], +['05', 'AR', 'ARKANSAS'], +['06', 'CA', 'CALIFORNIA'], +['08', 'CO', 'COLORADO'], +['09', 'CT', 'CONNECTICUT'], +['10', 'DE', 'DELAWARE'], +['11', 'DC', 'DISTRICT_OF_COLUMBIA'], +['12', 'FL', 'FLORIDA'], +['13', 'GA', 'GEORGIA'], +['15', 'HI', 'HAWAII'], +['16', 'ID', 'IDAHO'], +['17', 'IL', 'ILLINOIS'], +['18', 'IN', 'INDIANA'], +['19', 'IA', 'IOWA'], +['20', 'KS', 'KANSAS'], +['21', 'KY', 'KENTUCKY'], +['22', 'LA', 'LOUISIANA'], +['23', 'ME', 'MAINE'], +['24', 'MD', 'MARYLAND'], +['25', 'MA', 'MASSACHUSETTS'], +['26', 'MI', 'MICHIGAN'], +['27', 'MN', 'MINNESOTA'], +['28', 'MS', 'MISSISSIPPI'], +['29', 'MO', 'MISSOURI'], +['30', 'MT', 'MONTANA'], +['31', 'NE', 'NEBRASKA'], +['32', 'NV', 'NEVADA'], +['33', 'NH', 'NEW_HAMPSHIRE'], +['34', 'NJ', 'NEW_JERSEY'], +['35', 'NM', 'NEW_MEXICO'], +['36', 'NY', 'NEW_YORK'], +['37', 'NC', 'NORTH_CAROLINA'], +['38', 'ND', 'NORTH_DAKOTA'], +['39', 'OH', 'OHIO'], +['40', 'OK', 'OKLAHOMA'], +['41', 'OR', 'OREGON'], +['42', 'PA', 'PENNSYLVANIA'], +['72', 'PR', 'PUERTO_RICO'], +['44', 'RI', 'RHODE_ISLAND'], +['45', 'SC', 'SOUTH_CAROLINA'], +['46', 'SD', 'SOUTH_DAKOTA'], +['47', 'TN', 'TENNESSEE'], +['48', 'TX', 'TEXAS'], +['49', 'UT', 'UTAH'], +['50', 'VT', 'VERMONT'], +['51', 'VA', 'VIRGINIA'], +['53', 'WA', 'WASHINGTON'], +['54', 'WV', 'WEST_VIRGINIA'], +['55', 'WI', 'WISCONSIN'], +['56', 'WY', 'WYOMING'] +] + +# ======================================================================================================================= + + +def get_state_abb_from_state_fips(input_fips): + + return_value = 'XX' + + for state_data in STATES_DATA: + if state_data[0] == input_fips: + return_value = state_data[1] + + return return_value + +# ===================================================================================================================== + + +def assign_pipeline_costs(the_scenario, logger, include_pipeline): + scenario_gdb = os.path.join(the_scenario.scenario_run_directory, "main.gdb") + # calc the cost for pipeline + # --------------------- + + if include_pipeline: + logger.info("starting calculate cost for PIPELINE") + # base rate is cents per barrel + + # calculate for all + arcpy.MakeFeatureLayer_management (os.path.join(scenario_gdb, "pipeline"), "pipeline_lyr") + arcpy.SelectLayerByAttribute_management(in_layer_or_view="pipeline_lyr", selection_type="NEW_SELECTION", where_clause="Artificial = 0") + arcpy.CalculateField_management("pipeline_lyr", field="FROM_TO_ROUTING_COST", expression="(((!base_rate! / 100) / 42.0) * 1000.0)", expression_type="PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", field="FROM_TO_DOLLAR_COST", expression="(((!base_rate! / 100) / 42.0) * 1000.0)", expression_type="PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") + + # if base_rate is null, select those links that are not artifiical and set to 0 manually. + arcpy.SelectLayerByAttribute_management(in_layer_or_view="pipeline_lyr", selection_type="NEW_SELECTION", where_clause="Artificial = 0 and base_rate is null") + arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_ROUTING_COST", 0, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_DOLLAR_COST", 0, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") + no_baserate_count = int(arcpy.GetCount_management("pipeline_lyr").getOutput(0)) + if no_baserate_count > 0: + logger.warn("pipeline network contains {} routes with no base_rate. Flow cost will be set to 0".format(no_baserate_count)) + + else: + logger.info("starting calculate cost for PIPELINE = -1") + arcpy.MakeFeatureLayer_management (os.path.join(scenario_gdb, "pipeline"), "pipeline_lyr") + arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_ROUTING_COST", -1, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_ROUTING_COST", -1, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "FROM_TO_DOLLAR_COST", -1, "PYTHON_9.3") + arcpy.CalculateField_management("pipeline_lyr", "TO_FROM_DOLLAR_COST", -1, "PYTHON_9.3") + +# ====================================================================================================================== + + +def set_intermodal_links(the_scenario, logger): + scenario_gdb = the_scenario.main_gdb + # set the artificial link field to 2 so that it + # does not have flow restrictions applied to it. + + # assumes that the only artificial links in the network + # are connecting intermodal facilities. + + logger.info("start: set_intermodal_links") + + for mode in ["road", "rail", "pipeline_prod_trf_rts", "pipeline_crude_trf_rts", "water"]: + mode_layer = "{}_lyr".format(mode) + arcpy.MakeFeatureLayer_management(os.path.join(scenario_gdb, mode), mode_layer) + arcpy.SelectLayerByAttribute_management(in_layer_or_view=mode_layer, + selection_type="NEW_SELECTION", + where_clause="Artificial = 1") + arcpy.CalculateField_management(mode_layer, "ARTIFICIAL", 2, "PYTHON_9.3") + +# ====================================================================================================================== + + +class LicenseError(Exception): + pass + +# ====================================================================================================================== + + +def load_afpat_data_to_memory(fullPathToTable, logger): + + """load afpat data to memory""" + + logger.debug("START: load_afpat_data_to_memory") + + crop_yield_dict = {} # keyed off crop name --> crop yield (e.g. kg/ha/yr) + fuel_yield_dict = {} # keyed off crop name + processing type --> fuel yields (e.g. jet, diesel, total biomass) + + afpatAsList = [] + + # iterate through the afpat table and save as a list + # -------------------------------------------- + with arcpy.da.SearchCursor(fullPathToTable, "*") as cursor: + + for row in cursor: + + afpatAsList.append(row) + + # now identify the relevant indexs for the values we want to extract + + # land specific information + # ----------------------------------------- + countryIndex = afpatAsList[34].index('Country') + landAreaIndex = afpatAsList[34].index('Total Land Area Used') + + + # feedstock names + # ----------------------------------- + Feedstock_Source_Index = afpatAsList[34].index('Feedstock Source') + feedstock_type = afpatAsList[34].index('Feedstock Type') + feedstock_source_category = afpatAsList[34].index('Source Category') + + # crop yield index + #---------------------------- + oilIndex = afpatAsList[34].index('Oil Yield') + noncellulosicIndex = afpatAsList[34].index('Non-Cellulosic Sugar Yield') + lignoIndex = afpatAsList[34].index('Lignocellulosic Yield') + + # process index + # ------------------------------------------- + primaryProcTypeIndex = afpatAsList[34].index('Primary Processing Type') + secondaryProcTypeIndex = afpatAsList[34].index('Secondary Processing Type') + tertiaryProcTypeIndex = afpatAsList[34].index('Tertiary Processing Type') + capital_costs = afpatAsList[34].index('Capital Costs') + + # fuel yeilds index + # ------------------------------------------- + jetFuelIndex = afpatAsList[34].index('Jet Fuel ') + dieselFuelIndex = afpatAsList[34].index('Diesel fuel') + naphthaIndex = afpatAsList[34].index('Naphtha') + aromaticsIndex = afpatAsList[34].index('Aromatics') + total_fuel = afpatAsList[34].index('Total fuel (Excluding propane, LPG and heavy oil)') + conversion_efficiency = afpatAsList[34].index('Conversion Eff') + total_daily_biomass = afpatAsList[34].index('Total Daily Biomass') + + + table2cIndex = 0 + table2dIndex = 0 + + from ftot_supporting import get_cleaned_process_name + from ftot_supporting import create_full_crop_name + + for r in afpatAsList: + + # move biowaste to another table + if r[2] == "Table 2c:": + + table2cIndex = afpatAsList.index(r) + + # move fossil resources to another table + elif r[2] == "Table 2d:": + + table2dIndex = afpatAsList.index(r) + + # add commodity and process information to the crop_yield_dict and fuel_yield_dict + if r[countryIndex] == "U.S." and r[Feedstock_Source_Index] != "": + + # concatinate crop name + feedstock_name = create_full_crop_name(r[feedstock_type], r[feedstock_source_category],r[Feedstock_Source_Index]) + + + + # determine which crop yield field we want depending on the feedstock_type + if r[feedstock_type] == "Oils": + + preprocYield = float(r[oilIndex]) + + elif r[feedstock_type] == "Non-Cellulosic Sugars": + + preprocYield = float(r[noncellulosicIndex]) + + elif r[feedstock_type] == "Lignocellulosic Biomass and Cellulosic Sugars": + + preprocYield = float(r[lignoIndex]) + + else: + logger.error("Error in feedstock type specification from AFPAT") + raise Exception("Error in feedstock type specification from AFPAT") + + + # preprocYield is independent of processor type, so its okay + # if this gets written over by the next instance of the commodity + crop_yield_dict[feedstock_name] = preprocYield + + # Fuel Yield Dictionary + # --------------------------------- + # each commodity gets its own dictionary, keyed on the 3 process types; value is a list of the form + # [land area, jet, diesel, naphtha, aromatics, yield]; I'm adding total fuel and + # feedstock type to the end of this list; maybe units also? + + process_name = get_cleaned_process_name(r[primaryProcTypeIndex], r[secondaryProcTypeIndex],r[tertiaryProcTypeIndex]) + + if feedstock_name not in fuel_yield_dict: + + fuel_yield_dict[feedstock_name] = {process_name: [float(r[landAreaIndex]), + float(r[jetFuelIndex]), float(r[dieselFuelIndex]), + float(r[naphthaIndex]), float(r[aromaticsIndex]), + preprocYield, float(r[total_fuel]), + float(r[conversion_efficiency]), float(r[total_daily_biomass])]} + else: + + fuel_yield_dict[feedstock_name][process_name] = [float(r[landAreaIndex]), + float(r[jetFuelIndex]), float(r[dieselFuelIndex]), + float(r[naphthaIndex]), float(r[aromaticsIndex]), + preprocYield, float(r[total_fuel]), + float(r[conversion_efficiency]), float(r[total_daily_biomass])] + + table2c = list(itertools.islice(afpatAsList, table2cIndex + 2, table2dIndex)) + table2d = list(itertools.islice(afpatAsList, table2dIndex + 2, len(afpatAsList))) + + bioWasteDict = {} + fossilResources = {} + + resourceIndex = table2c[0].index("Resource") + procTypeIndex = table2c[0].index("Processing Type") + percentIndex = table2c[0].index("Percent of Resource") + resourceKgIndex = table2c[1].index("kg/yr") + facilitiesIndex = table2c[0].index("# FTx or HEFA Facilities") + capCostsIndex = table2c[0].index("Capital Costs") + totFuelIndex = table2c[0].index("Total fuel") + jetFuelIndex = table2c[0].index("Jet fuel") + dieselFuelIndex = table2c[0].index("Diesel fuel") + naphthaIndex = table2c[0].index("naphtha") + + numFtxIndex = table2d[0].index("# FTx Facilities") + ccsIndex = table2d[0].index("CCS Required") + eorIndex = table2d[0].index("Percent of EOR Capacity") + tabDcapCost = table2d[0].index("Capital Costs") + + secondaryProcType = "N/A" + tertiaryProcType = "N/A" + + + del table2c[0] + del table2d[0] + + for t in table2c: + + if t[2] != "": + + resourceName = t[resourceIndex].replace("-", "_").replace(" ", "_") + processType = (t[procTypeIndex].replace("-", "_").replace(" ", "_"), secondaryProcType, tertiaryProcType) + + if resourceName not in bioWasteDict: + + bioWasteDict[resourceName] = {processType: [float(t[percentIndex]), + float(t[resourceKgIndex]), + int(math.ceil(float( t[facilitiesIndex]))), + float(t[capCostsIndex]), + int(math.ceil(float( t[totFuelIndex]))), + int(math.ceil(float( t[jetFuelIndex]))), + int(math.ceil(float( t[dieselFuelIndex]))), + int(math.ceil(float( t[naphthaIndex])))]} + else: + + bioWasteDict[resourceName][processType] = [float(t[percentIndex]), + float(t[resourceKgIndex]), + int(math.ceil(float( t[facilitiesIndex]))), + float(t[capCostsIndex]), + int(math.ceil(float( t[totFuelIndex]))), + int(math.ceil(float( t[jetFuelIndex]))), + int(math.ceil(float( t[dieselFuelIndex]))), + int(math.ceil(float( t[naphthaIndex])))] + for t in table2d: + if t[2] != "" and t[3] != "": + + resourceName = t[resourceIndex].replace("-", "_").replace(" ", "_") + processType = (t[procTypeIndex].replace("-", "_").replace(" ", "_"), secondaryProcType, tertiaryProcType) + + if resourceName not in fossilResources: + + fossilResources[resourceName] = {processType: [int(t[numFtxIndex]), + float(t[tabDcapCost]), + float(t[ccsIndex]), + float(t[eorIndex]), + int(math.ceil(float( t[totFuelIndex]))), + int(math.ceil(float( t[jetFuelIndex]))), + int(math.ceil(float( t[dieselFuelIndex]))), + int(math.ceil(float( t[naphthaIndex])))]} + + + else: + + fossilResources[resourceName][processType] = [int(t[numFtxIndex]), + float(t[tabDcapCost]), + float(t[ccsIndex]), + float(t[eorIndex]), + int(math.ceil(float( t[totFuelIndex]))), + int(math.ceil(float( t[jetFuelIndex]))), + int(math.ceil(float( t[dieselFuelIndex]))), + int(math.ceil(float( t[naphthaIndex])))] + + logger.debug("FINISH: load_afpat_data_to_memory") + + return fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources +# ============================================================================== + + +def persist_AFPAT_tables(the_scenario, logger): + # pickle the AFPAT tables so it can be read in later without ArcPy + + scenario_gdb = the_scenario.main_gdb + + afpat_raw_table = os.path.join(scenario_gdb, "afpat_raw") + + fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources = load_afpat_data_to_memory(afpat_raw_table, logger) + + afpat_tables = [fuel_yield_dict, crop_yield_dict, bioWasteDict, fossilResources] + +# ============================================================================== + +# ********************************************************************** +# File name: Zip.py +# Description: +# Zips the contents of a folder, file geodatabase or ArcInfo workspace +# containing coverages into a zip file. +# Arguments: +# 0 - Input workspace +# 1 - Output zip file. A .zip extension should be added to the file name +# +# Created by: ESRI +# ********************************************************************** + +# Import modules and create the geoprocessor +import os + + +# Function for zipping files +def zipgdb(path, zip_file): + isdir = os.path.isdir + + # Check the contents of the workspace, if it the current + # item is a directory, gets its contents and write them to + # the zip file, otherwise write the current file item to the + # zip file + for each in os.listdir(path): + fullname = path + "/" + each + if not isdir(fullname): + # If the workspace is a file geodatabase, avoid writing out lock + # files as they are unnecessary + if not each.endswith('.lock'): + # Write out the file and give it a relative archive path + try: zip_file.write(fullname, each) + except IOError: None # Ignore any errors in writing file + else: + # Branch for sub-directories + for eachfile in os.listdir(fullname): + if not isdir(eachfile): + if not each.endswith('.lock'): + gp.AddMessage("Adding " + eachfile + " ...") + # Write out the file and give it a relative archive path + try: zip_file.write(fullname + "/" + eachfile, os.path.basename(fullname) + "/" + eachfile) + except IOError: None # Ignore any errors in writing file + + diff --git a/program/lib/Master_FTOT_Schema.xsd b/program/lib/Master_FTOT_Schema.xsd index 7713f63..c5a9843 100644 --- a/program/lib/Master_FTOT_Schema.xsd +++ b/program/lib/Master_FTOT_Schema.xsd @@ -1,391 +1,400 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/program/lib/detailed_emission_factors.csv b/program/lib/detailed_emission_factors.csv new file mode 100644 index 0000000..83d6d86 --- /dev/null +++ b/program/lib/detailed_emission_factors.csv @@ -0,0 +1,65 @@ +vehicle_label,mode,road_type,pollutant,value +Default,road,Rural_Restricted,CO,1.824 g/mi +Default,road,Rural_Unrestricted,CO,2.061 g/mi +Default,road,Urban_Restricted,CO,1.958 g/mi +Default,road,Urban_Unrestricted,CO,2.882 g/mi +Default,road,Rural_Restricted,CO2e,1622.368 g/mi +Default,road,Rural_Unrestricted,CO2e,1630.85 g/mi +Default,road,Urban_Restricted,CO2e,1617.818 g/mi +Default,road,Urban_Unrestricted,CO2e,1829.149 g/mi +Default,road,Rural_Restricted,CH4,0.039 g/mi +Default,road,Rural_Unrestricted,CH4,0.083 g/mi +Default,road,Urban_Restricted,CH4,0.062 g/mi +Default,road,Urban_Unrestricted,CH4,0.155 g/mi +Default,road,Rural_Restricted,N2O,0.001 g/mi +Default,road,Rural_Unrestricted,N2O,0.002 g/mi +Default,road,Urban_Restricted,N2O,0.002 g/mi +Default,road,Urban_Unrestricted,N2O,0.003 g/mi +Default,road,Rural_Restricted,NOx,3.925 g/mi +Default,road,Rural_Unrestricted,NOx,4.114 g/mi +Default,road,Urban_Restricted,NOx,3.998 g/mi +Default,road,Urban_Unrestricted,NOx,5.683 g/mi +Default,road,Rural_Restricted,PM10,0.131 g/mi +Default,road,Rural_Unrestricted,PM10,0.178 g/mi +Default,road,Urban_Restricted,PM10,0.159 g/mi +Default,road,Urban_Unrestricted,PM10,0.37 g/mi +Default,road,Rural_Restricted,PM2.5,0.088 g/mi +Default,road,Rural_Unrestricted,PM2.5,0.099 g/mi +Default,road,Urban_Restricted,PM2.5,0.094 g/mi +Default,road,Urban_Unrestricted,PM2.5,0.153 g/mi +Default,road,Rural_Restricted,VOC,0.181 g/mi +Default,road,Rural_Unrestricted,VOC,0.194 g/mi +Default,road,Urban_Restricted,VOC,0.188 g/mi +Default,road,Urban_Unrestricted,VOC,0.259 g/mi +small_truck,road,Rural_Restricted,CO,2.643 g/mi +small_truck,road,Rural_Unrestricted,CO,2.505 g/mi +small_truck,road,Urban_Restricted,CO,2.523 g/mi +small_truck,road,Urban_Unrestricted,CO,2.934 g/mi +small_truck,road,Rural_Restricted,CO2e,888.781 g/mi +small_truck,road,Rural_Unrestricted,CO2e,917.17 g/mi +small_truck,road,Urban_Restricted,CO2e,897.527 g/mi +small_truck,road,Urban_Unrestricted,CO2e,1099.56 g/mi +small_truck,road,Rural_Restricted,CH4,0.074 g/mi +small_truck,road,Rural_Unrestricted,CH4,0.107 g/mi +small_truck,road,Urban_Restricted,CH4,0.09 g/mi +small_truck,road,Urban_Unrestricted,CH4,0.197 g/mi +small_truck,road,Rural_Restricted,N2O,0.002 g/mi +small_truck,road,Rural_Unrestricted,N2O,0.003 g/mi +small_truck,road,Urban_Restricted,N2O,0.002 g/mi +small_truck,road,Urban_Unrestricted,N2O,0.005 g/mi +small_truck,road,Rural_Restricted,NOx,1.074 g/mi +small_truck,road,Rural_Unrestricted,NOx,1.276 g/mi +small_truck,road,Urban_Restricted,NOx,1.16 g/mi +small_truck,road,Urban_Unrestricted,NOx,2.006 g/mi +small_truck,road,Rural_Restricted,PM10,0.059 g/mi +small_truck,road,Rural_Unrestricted,PM10,0.097 g/mi +small_truck,road,Urban_Restricted,PM10,0.078 g/mi +small_truck,road,Urban_Unrestricted,PM10,0.214 g/mi +small_truck,road,Rural_Restricted,PM2.5,0.035 g/mi +small_truck,road,Rural_Unrestricted,PM2.5,0.044 g/mi +small_truck,road,Urban_Restricted,PM2.5,0.04 g/mi +small_truck,road,Urban_Unrestricted,PM2.5,0.073 g/mi +small_truck,road,Rural_Restricted,VOC,0.148 g/mi +small_truck,road,Rural_Unrestricted,VOC,0.174 g/mi +small_truck,road,Urban_Restricted,VOC,0.161 g/mi +small_truck,road,Urban_Unrestricted,VOC,0.258 g/mi diff --git a/program/lib/v6_temp_Scenario.xml b/program/lib/v6_temp_Scenario.xml index d2805f1..ce62849 100644 --- a/program/lib/v6_temp_Scenario.xml +++ b/program/lib/v6_temp_Scenario.xml @@ -1,170 +1,176 @@ - - - - 6.0.0 - USER INPUT REQUIRED: replace this string with a descriptive Scenario Name - USER INPUT REQUIRED: replace this string with a Scenario Description - - USER INPUT REQUIRED - - - - - USER INPUT REQUIRED - None - - - - USER INPUT REQUIRED - USER INPUT REQUIRED - USER INPUT REQUIRED - - - USER INPUT REQUIRED - USER INPUT REQUIRED - None - None - None - None - - - - - tonnes - kgal - - - - - 24 tonne - 82 tonne - 700 tonne - 8 kgal - 28.5 kgal - 2100 kgal - 3150 kgal - 3150 kgal - 7.4 mi/gal - 10.15 mi/gal - 5.00 mi/gal - - - 1550.19 g/mi - 1343.74 g/mi - 1360.18 g/mi - 1338.31 g/mi - - 21.0 g/ton/mi - 18.1 g/ton/mi - 0.0 g/ton/mi - - - - - - - 0.09 usd/kgal/mi - 0.03 usd/tonne/mi - - - - - - - 1.0 - 1.1 - 1.2 - 1.3 - 1.4 - 1.5 - 1.6 - 10.0 - - - 0.54 usd/kgal/mi - 0.19 usd/tonne/mi - - - - - - - 1.0 - 1.1 - 1.2 - 1.3 - - - 0.07 usd/kgal/mi - 0.02 usd/tonne/mi - - - - - - - - - 1.00 - 1.3 - 1.6 - 10 - - - - 40.00 usd/kgal - 12.35 usd/tonne - - - - - - - 5 mi - 5 mi - 5 mi - 5 mi - 5 mi - - - - - 61.2 usd/kgal - 21.7 usd/tonne - 63.2 usd/kgal - 22.7 usd/tonne - - - - False - - - - True - True - True - True - True - - - - - False - - - - False - False - False - False - False - - - - 0.00 - - - 5000 - - - + + + + 6.0.1 + USER INPUT REQUIRED: replace this string with a descriptive Scenario Name + USER INPUT REQUIRED: replace this string with a Scenario Description + + USER INPUT REQUIRED + + + + + USER INPUT REQUIRED + None + + + + USER INPUT REQUIRED + USER INPUT REQUIRED + USER INPUT REQUIRED + + + USER INPUT REQUIRED + USER INPUT REQUIRED + None + None + None + None + + + + + tonnes + kgal + + + + + 24 tonne + 82 tonne + 700 tonne + 8 kgal + 28.5 kgal + 2100 kgal + 3150 kgal + 3150 kgal + 7.4 mi/gal + 10.15 mi/gal + 5.00 mi/gal + + + 1550.19 g/mi + 1343.74 g/mi + 1360.18 g/mi + 1338.31 g/mi + + 21.0 g/ton/mi + 18.1 g/ton/mi + 0.0 g/ton/mi + + False + + + + + 3.33 ton/kgal + + + + + + + 0.14 usd/kgal/mi + 0.047 usd/tonne/mi + + + + + + + 1.0 + 1.1 + 1.2 + 1.3 + 1.4 + 1.5 + 1.6 + 10.0 + + + 0.66 usd/kgal/mi + 0.22 usd/tonne/mi + + + + + + + 1.0 + 1.1 + 1.2 + 1.3 + + + 0.097 usd/kgal/mi + 0.032 usd/tonne/mi + + + + + + + + + 1.00 + 1.3 + 1.6 + 10 + + + + 40.00 usd/kgal + 12.35 usd/tonne + + + + + + + 5 mi + 5 mi + 5 mi + 5 mi + 5 mi + + + + 71.8 usd/kgal + 23.9 usd/tonne + 76.1 usd/kgal + 25.4 usd/tonne + + + + False + + + + True + True + True + True + True + + + + + False + + + + False + False + False + False + False + + + + 0.00 + + + 5000 + + + \ No newline at end of file diff --git a/program/lib/vehicle_types.csv b/program/lib/vehicle_types.csv index 26b3b0b..1cc30bc 100644 --- a/program/lib/vehicle_types.csv +++ b/program/lib/vehicle_types.csv @@ -1,8 +1,8 @@ -vehicle_label,mode,vehicle_property,value -small_truck,road,Truck_Load_Solid,8 tonne -small_truck,road,Truck_Load_Liquid,2.5 kgal -small_truck,road,Truck_Fuel_Efficiency,12.1 mi/gal -small_truck,road,Atmos_CO2_Urban_Unrestricted,957.20 g/mi -small_truck,road,Atmos_CO2_Urban_Restricted,780.84 g/mi -small_truck,road,Atmos_CO2_Rural_Unrestricted,797.84 g/mi -small_truck,road,Atmos_CO2_Rural_Restricted,773.66 g/mi +vehicle_label,mode,vehicle_property,value +small_truck,road,Truck_Load_Solid,8 tonne +small_truck,road,Truck_Load_Liquid,2.5 kgal +small_truck,road,Truck_Fuel_Efficiency,12.1 mi/gal +small_truck,road,Atmos_CO2_Urban_Unrestricted,957.20 g/mi +small_truck,road,Atmos_CO2_Urban_Restricted,780.84 g/mi +small_truck,road,Atmos_CO2_Rural_Unrestricted,797.84 g/mi +small_truck,road,Atmos_CO2_Rural_Restricted,773.66 g/mi diff --git a/program/tools/facility_data_tools.py b/program/tools/facility_data_tools.py index 4985205..a98745b 100644 --- a/program/tools/facility_data_tools.py +++ b/program/tools/facility_data_tools.py @@ -1,124 +1,124 @@ -# -*- coding: utf-8 -*- -# the purpose of this code is to digest facility information from FTOT 3.1 and prior, -# and create csv data file for each facility. - -import argparse -import arcpy -import os - - - -if __name__ == '__main__': - print ("starting main") - debug=True - print ("debug state : {}".format(debug)) - - - # process destinations first - #--------------------------------------- - - # open csv file for writing - print ("opening a csv file") - output_file_name = "destination_demand.csv" - output_dir = "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data" - output_file = os.path.join(output_dir, output_file_name) - with open(output_file, 'w') as wf: - - # write the header line - header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" - wf.write(str(header_line+"\n")) - - # search cursor on the featureclass - fields = ["facility_name", "Demand_Jet"] - fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\ultimate_destinations" - print ("opening fc: {}".format(fc)) - with arcpy.da.SearchCursor(fc, fields) as cursor: - - for row in cursor: - if debug: print ("processing row: {}".format(row)) - facility_name = row[0] - facility_type = "ultimate_destination" - commodity = "jet" - value = row[1] - units = "kgal" - phase_of_matter = "liquid" - io = "i" - - # csv writer.write(row) - if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) - wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) - - - - - # process rmps - #-------------------- - print ("opening a csv file") - - output_file_name = "rmp_supply.csv" - output_dir = "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data" - output_file = os.path.join(output_dir, output_file_name) - with open(output_file, 'w') as wf: - - # write the header line - rmp_header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,max_transport_distance,io" - header_line = rmp_header_line - wf.write(str(header_line+"\n")) - - # search cursor on the featureclass - fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\raw_material_producers" - fields=["facility_name", "Demand_Jet"] - print ("opening fc: {}".format(fc)) - with arcpy.da.SearchCursor(fc, fields) as cursor: - - for row in cursor: - if debug: print ("processing row: {}".format(row)) - facility_name = row[0] - facility_type = "ultimate_destination" - commodity = "jet" - value = row[1] - units = "kgal" - phase_of_matter = "liquid" - io = "i" - - # csv writer.write(row) - if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) - wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) - - -def process_rmp_fc(debug=False): - - print ("debug state : {}".format(debug)) - - - # open csv file for writing - print ("opening a csv file") - output_file = os.path.join(output_dir, output_file_name) - with open(output_file, 'w') as wf: - - # write the header line - rmp_header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,max_transport_distance,io" - header_line = rmp_header_line - wf.write(str(header_line+"\n")) - - # search cursor on the featureclass - fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\raw_material_producers" - fields=[""] - print ("opening fc: {}".format(fc)) - with arcpy.da.SearchCursor(fc, fields) as cursor: - - for row in cursor: - if debug: print ("processing row: {}".format(row)) - facility_name = row[0] - facility_type = "ultimate_destination" - commodity = "jet" - value = row[1] - units = "kgal" - phase_of_matter = "liquid" - io = "i" - - # csv writer.write(row) - if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) - wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) - - +# -*- coding: utf-8 -*- +# the purpose of this code is to digest facility information from FTOT 3.1 and prior, +# and create csv data file for each facility. + +import argparse +import arcpy +import os + + + +if __name__ == '__main__': + print ("starting main") + debug=True + print ("debug state : {}".format(debug)) + + + # process destinations first + #--------------------------------------- + + # open csv file for writing + print ("opening a csv file") + output_file_name = "destination_demand.csv" + output_dir = "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data" + output_file = os.path.join(output_dir, output_file_name) + with open(output_file, 'w') as wf: + + # write the header line + header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" + wf.write(str(header_line+"\n")) + + # search cursor on the featureclass + fields = ["facility_name", "Demand_Jet"] + fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\ultimate_destinations" + print ("opening fc: {}".format(fc)) + with arcpy.da.SearchCursor(fc, fields) as cursor: + + for row in cursor: + if debug: print ("processing row: {}".format(row)) + facility_name = row[0] + facility_type = "ultimate_destination" + commodity = "jet" + value = row[1] + units = "kgal" + phase_of_matter = "liquid" + io = "i" + + # csv writer.write(row) + if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) + wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) + + + + + # process rmps + #-------------------- + print ("opening a csv file") + + output_file_name = "rmp_supply.csv" + output_dir = "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data" + output_file = os.path.join(output_dir, output_file_name) + with open(output_file, 'w') as wf: + + # write the header line + rmp_header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,max_transport_distance,io" + header_line = rmp_header_line + wf.write(str(header_line+"\n")) + + # search cursor on the featureclass + fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\raw_material_producers" + fields=["facility_name", "Demand_Jet"] + print ("opening fc: {}".format(fc)) + with arcpy.da.SearchCursor(fc, fields) as cursor: + + for row in cursor: + if debug: print ("processing row: {}".format(row)) + facility_name = row[0] + facility_type = "ultimate_destination" + commodity = "jet" + value = row[1] + units = "kgal" + phase_of_matter = "liquid" + io = "i" + + # csv writer.write(row) + if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) + wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) + + +def process_rmp_fc(debug=False): + + print ("debug state : {}".format(debug)) + + + # open csv file for writing + print ("opening a csv file") + output_file = os.path.join(output_dir, output_file_name) + with open(output_file, 'w') as wf: + + # write the header line + rmp_header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,max_transport_distance,io" + header_line = rmp_header_line + wf.write(str(header_line+"\n")) + + # search cursor on the featureclass + fc= "C:\\FTOT\\branches\\2017_08_31_processor\\scenarios\\USDA_oil_seed_dec_2017\\data\\rmp_data_ftot_v3_2.gdb\\network\\raw_material_producers" + fields=[""] + print ("opening fc: {}".format(fc)) + with arcpy.da.SearchCursor(fc, fields) as cursor: + + for row in cursor: + if debug: print ("processing row: {}".format(row)) + facility_name = row[0] + facility_type = "ultimate_destination" + commodity = "jet" + value = row[1] + units = "kgal" + phase_of_matter = "liquid" + io = "i" + + # csv writer.write(row) + if debug: print ("writing airport: {} and demand: {} \t {}".format(facility_name, value, units)) + wf.write("{},{},{},{},{},{},{}\n".format(facility_name,facility_type,commodity,value,units,phase_of_matter,io)) + + diff --git a/program/tools/ftot_tools.py b/program/tools/ftot_tools.py index e5efde2..9b89525 100644 --- a/program/tools/ftot_tools.py +++ b/program/tools/ftot_tools.py @@ -1,101 +1,101 @@ -import os -import lxml_upgrade_tool -import run_upgrade_tool -import input_csv_templates_tool -import scenario_compare_tool -import gridded_data_tool -import xml_text_replacement_tool -import network_disruption_tool -from six.moves import input - -header = "\n\ - _______ _______ _______ _______ _______ _______ _______ ___ _______\n\ -| || || || | | || || || | | |\n\ -| ___||_ _|| _ ||_ _| |_ _|| _ || _ || | | _____|\n\ -| |___ | | | | | | | | | | | | | || | | || | | |_____ \n\ -| ___| | | | |_| | | | | | | |_| || |_| || |___ |_____ |\n\ -| | | | | | | | | | | || || | _____| |\n\ -|___| |___| |_______| |___| |___| |_______||_______||_______||_______|\n" - - -def xml_tool(): - print("You called xml_tool()") - xml_file_location = lxml_upgrade_tool.repl() - - -def bat_tool(): - print("You called bat_tool()") - run_upgrade_tool.run_bat_upgrade_tool() - input("Press [Enter] to continue...") - - -def compare_tool(): - print("You called compare_tool()") - scenario_compare_tool.run_scenario_compare_prep_tool() - input("Press [Enter] to continue...") - - -def raster_tool(): - print("You called aggregate_raster_data()") - gridded_data_tool.run() - input("Press [Enter] to continue...") - - -def csv_tool(): - print("You called csv_tool()") - input_csv_templates_tool.run_input_csv_templates_tool() - input("Press [Enter] to continue...") - - -def pdb(): - print("You called pdb()") - import pdb; pdb.set_trace() - input("Press [Enter] to continue...") - - -def replace_xml_text_tool(): - print("You called replace_xml_text()") - xml_text_replacement_tool.run() - input("Press [Enter] to continue...") - - -def disrupt_tool(): - print("You called network_disruption_tool()") - network_disruption_tool.run_network_disruption_tool() - input("Press [Enter] to continue...") - - -menuItems = [ - {"xml_tool": xml_tool}, - {"bat_tool": bat_tool}, - {"scenario_compare_tool": compare_tool}, - {"aggregate_raster_data": raster_tool}, - {"generate_template_csv_files": csv_tool}, - {"replace_xml_text": replace_xml_text_tool}, - {"network_disruption_tool": disrupt_tool}, - {"breakpoint": pdb}, - {"exit": exit} -] - - -def main(): - while True: - os.system('cls') - print(header) - print('version 0.1\n') - print('select an option below to activate a tool') - print('-----------------------------------------') - for item in menuItems: - print("[" + str(menuItems.index(item)) + "] " + list(item.keys())[0]) - choice = input(">> ") - try: - if int(choice) < 0: - raise ValueError - # Call the matching function - list(menuItems[int(choice)].values())[0]() - except (ValueError, IndexError): - pass - - -if __name__ == "__main__": - main() +import os +import lxml_upgrade_tool +import run_upgrade_tool +import input_csv_templates_tool +import scenario_compare_tool +import gridded_data_tool +import xml_text_replacement_tool +import network_disruption_tool +from six.moves import input + +header = "\n\ + _______ _______ _______ _______ _______ _______ _______ ___ _______\n\ +| || || || | | || || || | | |\n\ +| ___||_ _|| _ ||_ _| |_ _|| _ || _ || | | _____|\n\ +| |___ | | | | | | | | | | | | | || | | || | | |_____ \n\ +| ___| | | | |_| | | | | | | |_| || |_| || |___ |_____ |\n\ +| | | | | | | | | | | || || | _____| |\n\ +|___| |___| |_______| |___| |___| |_______||_______||_______||_______|\n" + + +def xml_tool(): + print("You called xml_tool()") + xml_file_location = lxml_upgrade_tool.repl() + + +def bat_tool(): + print("You called bat_tool()") + run_upgrade_tool.run_bat_upgrade_tool() + input("Press [Enter] to continue...") + + +def compare_tool(): + print("You called compare_tool()") + scenario_compare_tool.run_scenario_compare_prep_tool() + input("Press [Enter] to continue...") + + +def raster_tool(): + print("You called aggregate_raster_data()") + gridded_data_tool.run() + input("Press [Enter] to continue...") + + +def csv_tool(): + print("You called csv_tool()") + input_csv_templates_tool.run_input_csv_templates_tool() + input("Press [Enter] to continue...") + + +def pdb(): + print("You called pdb()") + import pdb; pdb.set_trace() + input("Press [Enter] to continue...") + + +def replace_xml_text_tool(): + print("You called replace_xml_text()") + xml_text_replacement_tool.run() + input("Press [Enter] to continue...") + + +def disrupt_tool(): + print("You called network_disruption_tool()") + network_disruption_tool.run_network_disruption_tool() + input("Press [Enter] to continue...") + + +menuItems = [ + {"xml_tool": xml_tool}, + {"bat_tool": bat_tool}, + {"scenario_compare_tool": compare_tool}, + {"aggregate_raster_data": raster_tool}, + {"generate_template_csv_files": csv_tool}, + {"replace_xml_text": replace_xml_text_tool}, + {"network_disruption_tool": disrupt_tool}, + {"breakpoint": pdb}, + {"exit": exit} +] + + +def main(): + while True: + os.system('cls') + print(header) + print('version 0.1\n') + print('select an option below to activate a tool') + print('-----------------------------------------') + for item in menuItems: + print("[" + str(menuItems.index(item)) + "] " + list(item.keys())[0]) + choice = input(">> ") + try: + if int(choice) < 0: + raise ValueError + # Call the matching function + list(menuItems[int(choice)].values())[0]() + except (ValueError, IndexError): + pass + + +if __name__ == "__main__": + main() diff --git a/program/tools/gridded_data_tool.py b/program/tools/gridded_data_tool.py index df536db..052b24b 100644 --- a/program/tools/gridded_data_tool.py +++ b/program/tools/gridded_data_tool.py @@ -1,112 +1,112 @@ -import arcpy -import os -from six.moves import input - -# THIS SCRIPT IS USED TO AGGREGATE GRID CELL PRODUCTION DATA, E.G., FBEP OR USDA, BY COUNTY -countyLyr = r"C:\FTOT\scenarios\common_data\base_layers\cb_2017_us_county_500k.shp" - - -def get_user_input(): - print("start: get_user_input") - return input(">>> input raster: ") - - -def cleanup(gdb): - print("start: cleanup") - if arcpy.Exists(gdb): - print("start: delete existing gdb") - print("gdb file location: " + gdb) - try: - arcpy.Delete_management(gdb) - - except: - print ("Couldn't delete " + gdb) - raise Exception("Couldn't delete " + gdb) - - -def aggregate_raster(): - - """aggregates the rasters by county""" - print("start: aggregate_raster") - inputRaster = get_user_input() - outFolder = os.path.dirname(inputRaster) - raster_name = str(os.path.basename(inputRaster)) - outGdbName = str(raster_name + ".gdb") - tempGdb = os.path.join(outFolder, outGdbName) - outputCounties = str(os.path.join(tempGdb,raster_name)) - - cleanup(tempGdb) - if not arcpy.Exists(tempGdb): - arcpy.CreateFileGDB_management(outFolder, outGdbName) - outPoints = os.path.join(tempGdb, "outPoints") - clippedRaster = os.path.join(tempGdb, "clippedFBEP") - joinOutput = os.path.join(tempGdb, "joinOutput") - - # 5 arc-minute is about 10 square kilometers - sqKilometersPer5ArcMinute = 10 - hectaresPerSqKilometer = 100 - hectareArea = sqKilometersPer5ArcMinute * hectaresPerSqKilometer - - rectangle = "-124.848974 24.396308 -66.885444 49.384358" - print("start: clip_management") - arcpy.Clip_management(inputRaster, rectangle, clippedRaster) - - print("start: raster_to_point_conversion") - arcpy.RasterToPoint_conversion(clippedRaster, outPoints) - - print("start: project_management") - arcpy.Project_management(countyLyr, os.path.join(tempGdb, "counties_projected"), arcpy.Describe(outPoints).spatialReference) - - print("start: spatial_join") - arcpy.SpatialJoin_analysis(outPoints, os.path.join(tempGdb, "counties_projected"), joinOutput, "JOIN_ONE_TO_ONE", - "KEEP_COMMON", "", "COMPLETELY_WITHIN") - - #countyProdDict[FIPS] = total tons - countyProdDict = {} - print("start: raster_output") - print("start: ...\\create_county_production_dictionary loop") - for row in arcpy.da.SearchCursor(joinOutput, ["GEOID", "grid_code"]): - if row[0] not in countyProdDict: - countyProdDict[row[0]] = row[1] * hectareArea - else: - countyProdDict[row[0]] += row[1] * hectareArea - - print("start: ...\\create_output_counties_fc") - arcpy.CopyFeatures_management(countyLyr, outputCounties) - - print("start: ...\\add_fields_to_county_output_fc") - arcpy.AddField_management(outputCounties, "Facility_Name", "TEXT") - - print("start: ...\\calculate_field_for_facility_name") - arcpy.CalculateField_management(in_table=outputCounties, field="Facility_Name", - expression="'rmp_'+!GEOID!", expression_type="PYTHON_9.3", code_block="") - - print("start: create_csv") - filename = str("rmp_" + commodity + ".csv") - facility_type = "raw_material_producer" - units = "kg" - phase_of_matter = "solid" - - a_filename = os.path.join(outFolder, filename) - with open(a_filename, 'w') as wf: - # write the header line - header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" - wf.write(str(header_line + "\n")) - for key in countyProdDict: - record = "{},{},{},{},{},{},{}".format(str("rmp_"+key), facility_type, raster_name, countyProdDict[key], - units, - phase_of_matter, 'o') - wf.write(str(record + "\n")) - - print("start: delete_temp_layers") - arcpy.Delete_management(outPoints) - arcpy.Delete_management(clippedRaster) - arcpy.Delete_management(joinOutput) - arcpy.Delete_management(counties_projected) - - print("finish: aggregate_raster") - return - - -def run(): - aggregate_raster() +import arcpy +import os +from six.moves import input + +# THIS SCRIPT IS USED TO AGGREGATE GRID CELL PRODUCTION DATA, E.G., FBEP OR USDA, BY COUNTY +countyLyr = r"C:\FTOT\scenarios\common_data\base_layers\cb_2017_us_county_500k.shp" + + +def get_user_input(): + print("start: get_user_input") + return input(">>> input raster: ") + + +def cleanup(gdb): + print("start: cleanup") + if arcpy.Exists(gdb): + print("start: delete existing gdb") + print("gdb file location: " + gdb) + try: + arcpy.Delete_management(gdb) + + except: + print ("Couldn't delete " + gdb) + raise Exception("Couldn't delete " + gdb) + + +def aggregate_raster(): + + """aggregates the rasters by county""" + print("start: aggregate_raster") + inputRaster = get_user_input() + outFolder = os.path.dirname(inputRaster) + raster_name = str(os.path.basename(inputRaster)) + outGdbName = str(raster_name + ".gdb") + tempGdb = os.path.join(outFolder, outGdbName) + outputCounties = str(os.path.join(tempGdb,raster_name)) + + cleanup(tempGdb) + if not arcpy.Exists(tempGdb): + arcpy.CreateFileGDB_management(outFolder, outGdbName) + outPoints = os.path.join(tempGdb, "outPoints") + clippedRaster = os.path.join(tempGdb, "clippedFBEP") + joinOutput = os.path.join(tempGdb, "joinOutput") + + # 5 arc-minute is about 10 square kilometers + sqKilometersPer5ArcMinute = 10 + hectaresPerSqKilometer = 100 + hectareArea = sqKilometersPer5ArcMinute * hectaresPerSqKilometer + + rectangle = "-124.848974 24.396308 -66.885444 49.384358" + print("start: clip_management") + arcpy.Clip_management(inputRaster, rectangle, clippedRaster) + + print("start: raster_to_point_conversion") + arcpy.RasterToPoint_conversion(clippedRaster, outPoints) + + print("start: project_management") + arcpy.Project_management(countyLyr, os.path.join(tempGdb, "counties_projected"), arcpy.Describe(outPoints).spatialReference) + + print("start: spatial_join") + arcpy.SpatialJoin_analysis(outPoints, os.path.join(tempGdb, "counties_projected"), joinOutput, "JOIN_ONE_TO_ONE", + "KEEP_COMMON", "", "COMPLETELY_WITHIN") + + #countyProdDict[FIPS] = total tons + countyProdDict = {} + print("start: raster_output") + print("start: ...\\create_county_production_dictionary loop") + for row in arcpy.da.SearchCursor(joinOutput, ["GEOID", "grid_code"]): + if row[0] not in countyProdDict: + countyProdDict[row[0]] = row[1] * hectareArea + else: + countyProdDict[row[0]] += row[1] * hectareArea + + print("start: ...\\create_output_counties_fc") + arcpy.CopyFeatures_management(countyLyr, outputCounties) + + print("start: ...\\add_fields_to_county_output_fc") + arcpy.AddField_management(outputCounties, "Facility_Name", "TEXT") + + print("start: ...\\calculate_field_for_facility_name") + arcpy.CalculateField_management(in_table=outputCounties, field="Facility_Name", + expression="'rmp_'+!GEOID!", expression_type="PYTHON_9.3", code_block="") + + print("start: create_csv") + filename = str("rmp_" + commodity + ".csv") + facility_type = "raw_material_producer" + units = "kg" + phase_of_matter = "solid" + + a_filename = os.path.join(outFolder, filename) + with open(a_filename, 'w') as wf: + # write the header line + header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" + wf.write(str(header_line + "\n")) + for key in countyProdDict: + record = "{},{},{},{},{},{},{}".format(str("rmp_"+key), facility_type, raster_name, countyProdDict[key], + units, + phase_of_matter, 'o') + wf.write(str(record + "\n")) + + print("start: delete_temp_layers") + arcpy.Delete_management(outPoints) + arcpy.Delete_management(clippedRaster) + arcpy.Delete_management(joinOutput) + arcpy.Delete_management(counties_projected) + + print("finish: aggregate_raster") + return + + +def run(): + aggregate_raster() diff --git a/program/tools/input_csv_templates_tool.py b/program/tools/input_csv_templates_tool.py index b5d66d0..2d48e90 100644 --- a/program/tools/input_csv_templates_tool.py +++ b/program/tools/input_csv_templates_tool.py @@ -1,50 +1,50 @@ -# run.bat upgrade tool - -# the run.bat upgrade tool is used to generate run.bat files for the user. -# it is a command-line-interface (CLI) tool that prompts the user for the following information: -# - python installation (32 or 64 bit) -# - FTOT program directory -# - Scenario XML File -# - Candidate generation? Y/N -# - Output file location -# ============================================================================== -import os -from six.moves import input - -# ============================================================================== - - -def run_input_csv_templates_tool(): - print("FTOT input csv template generation tool") - print("-----------------------------------------") - print("") - print("") - generate_input_csv_templates() - - -def generate_input_csv_templates(): - - print("start: generate_input_csv_templates") - template_list = ["rmp", "proc", "proc_cand", "dest"] - - input_data_dir = get_user_configs() - for facility in template_list: - a_filename = os.path.join(input_data_dir, facility+".csv") - print("opening a csv file for {}: here -- {}".format(facility, a_filename)) - with open(a_filename, 'w') as wf: - # write the header line - header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" - wf.write(str(header_line + "\n")) - - -def get_user_configs(): - # user specified input_data directory - print("directory to store the template CSV files: (drag and drop is fine here)") - input_data_dir = "" - while not os.path.exists(input_data_dir): - input_data_dir = input('----------------------> ') - print("USER INPUT ----------------->: {}".format(input_data_dir)) - if not os.path.exists(input_data_dir): - print("the following path is not valid. Please enter a valid path to python.exe") - print("os.path.exists == False for: {}".format(input_data_dir)) - return input_data_dir +# run.bat upgrade tool + +# the run.bat upgrade tool is used to generate run.bat files for the user. +# it is a command-line-interface (CLI) tool that prompts the user for the following information: +# - python installation (32 or 64 bit) +# - FTOT program directory +# - Scenario XML File +# - Candidate generation? Y/N +# - Output file location +# ============================================================================== +import os +from six.moves import input + +# ============================================================================== + + +def run_input_csv_templates_tool(): + print("FTOT input csv template generation tool") + print("-----------------------------------------") + print("") + print("") + generate_input_csv_templates() + + +def generate_input_csv_templates(): + + print("start: generate_input_csv_templates") + template_list = ["rmp", "proc", "proc_cand", "dest"] + + input_data_dir = get_user_configs() + for facility in template_list: + a_filename = os.path.join(input_data_dir, facility+".csv") + print("opening a csv file for {}: here -- {}".format(facility, a_filename)) + with open(a_filename, 'w') as wf: + # write the header line + header_line = "facility_name,facility_type,commodity,value,units,phase_of_matter,io" + wf.write(str(header_line + "\n")) + + +def get_user_configs(): + # user specified input_data directory + print("directory to store the template CSV files: (drag and drop is fine here)") + input_data_dir = "" + while not os.path.exists(input_data_dir): + input_data_dir = input('----------------------> ') + print("USER INPUT ----------------->: {}".format(input_data_dir)) + if not os.path.exists(input_data_dir): + print("the following path is not valid. Please enter a valid path to python.exe") + print("os.path.exists == False for: {}".format(input_data_dir)) + return input_data_dir diff --git a/program/tools/lxml_upgrade_tool.py b/program/tools/lxml_upgrade_tool.py index 60e3e41..643b819 100644 --- a/program/tools/lxml_upgrade_tool.py +++ b/program/tools/lxml_upgrade_tool.py @@ -1,346 +1,346 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Nov 05 13:05:27 2018 - -@author: Matthew.Pearlson.CTR -""" - -# lxml text code - -# read in a template XML document that would pass the schema validation. -# for now, this is just template for the tes, and in the future, we could -# have the template XML generated by the latest XSD in the lib folder. - -import os -import sys -from six.moves import input - -try: - from lxml import etree, objectify -except ImportError: - print ("This script requires the lxml Python library to validate the XML scenario file.") - print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") - print("Exiting...") - sys.exit() - - -# ====================================================================================================================== - -# define the location of the template globally -# we reverse out the lib folder from the local tools module path - -tools_dir = os.path.split(os.path.realpath(__file__))[0] -ftot_program_directory = os.path.split(tools_dir)[0] -xml_template_file_location = os.path.join(ftot_program_directory, "lib", "v6_temp_Scenario.xml") - - - -def repl_old(): - - should_quit = False - - xml_file_location = "" - while not should_quit: - print ("XML Utility Options") - print ("-------------------") - print ("(1) ------ Create new template with defaults") - print ("(2) ------ Upgrade an old XML") - print ("(3) ------ Quit....") - ux_response = get_user_input("enter a selection") - if ux_response == "1": - print("XML task") - # (1) raw_input() on default XML - # output a fully commented XML template for the user to fill out manually - xml_file_location = generate_xml() - elif ux_response == "2": - # (2) upgrade xml - # compare an old XML file with a new tool - xml_file_location = xml_upgrade_tool() - elif ux_response == "3": - should_quit = True - else: - print("warning: response: {} -- not recognized. please enter a valid value: 1, 2, 3".format(ux_response)) - - return xml_file_location -# ====================================================================================================================== - - -def load_scenario_config_file(fullPathToXmlConfigFile): - - if not os.path.exists(fullPathToXmlConfigFile): - raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) - - if fullPathToXmlConfigFile.rfind(".xml") < 0: - raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) - - parser = etree.XMLParser(remove_blank_text=True) - return etree.parse(fullPathToXmlConfigFile, parser) - -# ============================================================================== - - -def do_the_upgrades(the_template_etree, the_old_xml_etree): - - # load both documents to the lxml etree - # iterate through the template - # -------------------------------------------------------------- - item_counter = 0 - for temp_elem in the_template_etree.getiterator(): - item_counter += 1 - - # check if the temp_element has the .find attribute - # if it does, then search for the index - # to '{' char at the end of namespace (e.g. {FTOT}Scenario) - # if there is no attribute, we continue b/c its probably a comment. - if not hasattr(temp_elem.tag, 'find'): - continue - clean_temp_elem = clean_element_name(temp_elem) - - # if the text in the element is just white space (ignore it) - # -------------------------------------------------------------- - if temp_elem.text == "": - continue - - # otherwise, inspect the element. - - # now we are going to iterate through the old XML and try to find a - # matching records. If we find one, we'll flip the set_flag variable - # (intialized below) to True, to indicate we found one. If we get to - # the end of the old XML loop and there is no match, we query the user. - # ---------------------------------------------------------------------- - set_flag = False - - for old_elem in the_old_xml_etree.getiterator(): - - # do a little bit of clean up on the namespaces... - - # first make sure its not a comment, - # by checking if it has the ".find" attribute - # ------------------------------------------- - if not hasattr(old_elem.tag, 'find'): continue # (1) - - # find the index of the end of the namespace - # then clean the name so there are no namespaces - # to mess with the compare - # ------------------------------------------- - clean_old_elem = clean_element_name(old_elem) - - # check the cleaned tags for a match - # ------------------------------------------- - - # a matched tag scenario - if clean_temp_elem == clean_old_elem: - # get the parent element name - temp_parent = temp_elem.getparent() - old_parent = old_elem.getparent() - # check if the parent Null - clean_temp_parent = None - clean_old_parent = None - if temp_parent is not None: - clean_temp_parent = clean_element_name(temp_parent) - if old_parent is not None: - clean_old_parent = clean_element_name(old_parent) - - if clean_temp_parent != clean_old_parent: - # if the parents don't match, keep iterating. - continue - - # check if there is a "default response" - # --------------------------------------- - ux_response = is_a_default_new_tag(clean_temp_elem) - if ux_response == "": - ux_script = "which one do you want keep? enter: old, new, edit, or quit" - ux_response = get_user_input(ux_script) - - # if not, ask the user which one to keep - # --------------------------------------- - if ux_response == "old": - temp_elem.text = old_elem.text - set_flag = True - elif ux_response == "new": - set_flag = True - elif ux_response == "edit": - temp_elem.text = get_user_input("enter a value press [return]") - set_flag = True - elif ux_response in ["quit", "exit"]: - io_error = "user asked to quit...quiting without saving" - raise IOError(io_error) - else: - print("warning: '{}' is not a valid input. try again... or quit".format(ux_response)) - - # a no match scenario. - # --------------------- - else: - continue - - # if at the end of the loop we didnt match an old element - # against the new template, get input from the user - # current default behavior is to keep the - # new template values as the default - if not set_flag: - - # accept the template element as the default value - # ------------------------------------------------- - temp_elem.text = temp_elem.text # this doesnt actually do anything - - # uncomment to allow the user to specify the new tag - # ---------------------------------------------------- - #ux_string = "The temp_element {} -- didn't match against the old xml file... it needs a value \n temp_xml value: {}".format(clean_temp_elem, temp_elem.text) - #temp_elem.text = get_user_input(ux_string) - - # record the change in the log - print("setting temp_elem: {} to {}".format(clean_temp_elem, temp_elem.text)) - - return - -# ============================================================================== - - -def record_deprecated_elements(): - pass - # now, check for deprecated elements by looping through the old document - # for each elemet in the old document - # check if there is a match in the new document. - # if there is not, then log it! - - -# ============================================================================== - - -def is_a_default_new_tag(clean_temp_elem): - old_or_new = "" - - default_new_list = [ - "Scenario_Schema_Version" - ] - default_old_list = [ - "Scenario_Name", - "Scenario_Description", - "Common_Data_Folder", - "Base_Network_Gdb", - "Base_RMP_Layer" - ] - - if clean_temp_elem in default_new_list: old_or_new = "new" - else: old_or_new = "old" - - return old_or_new - -# ============================================================================== - - -def clean_element_name(element): - # REMOVE THE NAMESPACE FROM THE FRONT OF THE TAG - # e.g. {FTOT}Scenario --> Scenario - # find the index of the end of the namespace - # then clean the name so there are no namespaces - # to mess with the compare - #------------------------------------------- - ns_i = element.tag.find("}") - return element.tag[ns_i+1:] - -# ============================================================================== - - -def get_user_input(ux_string): - # ask the user for a XML file please - print (ux_string) - user_input = input('--> ') - print("user_input: {}".format(user_input)) - return user_input - -# ============================================================================== - - -def save_the_new_xml_file(the_temp_etree): - report_file_dir = False - should_quit = False - while not should_quit: - report_file_dir = get_user_input("enter a path to save the scenario file") - - if os.path.exists(str(report_file_dir)):should_quit = True - if report_file_dir == "quit": should_quit = True - if report_file_dir == False: print("warning: report file dir {} not found. enter a new path or type 'quit'".format(report_file_dir)) - - report_file_name = get_user_input("name of the scenario file") - if report_file_name .rfind(".xml") < 0: - report_file_name += ".xml" - - report_file = os.path.join(report_file_dir, report_file_name) - - with open(report_file, 'wb') as wf: - print("writing the file: {} ".format(report_file)) - the_temp_etree.write(wf, pretty_print=True) - print("done writing xml file: {}".format(report_file)) - - return report_file - -# ============================================================================== - - -def generate_xml(): - # copies the template xml file with default values - # from the lib folder to a location the user specifies - - the_temp_etree = load_scenario_config_file(xml_template_file_location) - return save_the_new_xml_file(the_temp_etree) - - -# ============================================================================== - - -def xml_upgrade_tool(): - - CLI_UX_TEMP_XML_PATH = "give me the template please...drag and drop is fine here" - - the_temp_etree = load_scenario_config_file(xml_template_file_location) -# # ask the user for a XML file please -# print ("give me the template please...drag and drop is fine here") -# the_template_path = raw_input('--> ') -# print("USER INPUT: the_template_path: {}".format(the_template_path)) -# the_temp_etree = load_scenario_config_file(the_template_path) - - - # ask the user for a XML file please - print ("give me the XML to upgrade please...drag and drop is fine here") - the_old_xml_path = input('--> ') - print("USER INPUT: the_old_xml_path: {}".format(the_old_xml_path)) - the_old_xml_etree = load_scenario_config_file(the_old_xml_path) - - print("about to do the upgrade analysis") - do_the_upgrades(the_temp_etree, the_old_xml_etree) - - print("saving the xml template as a new file") - return save_the_new_xml_file(the_temp_etree) - -# ============================================================================== - - -def should_quit(): - print ("returning to the FTOT Tools REPL") - return False - - -# ============================================================================== - - -menuItems = [ - { "create new template with default values": generate_xml}, - { "upgrade an old XML": xml_upgrade_tool}, - { "exit": should_quit }, -] - -def repl(): - stay_in_repl = True - os.system('cls') - while stay_in_repl: - print ("XML Tools") - for item in menuItems: - print("[" + str(menuItems.index(item)) + "] " + list(item.keys())[0]) - choice = input(">> ") - try: - if int(choice) < 0 : raise ValueError - # Call the matching function - stay_in_repl = list(menuItems[int(choice)].values())[0]() - except (ValueError, IndexError): - pass +# -*- coding: utf-8 -*- +""" +Created on Mon Nov 05 13:05:27 2018 + +@author: Matthew.Pearlson.CTR +""" + +# lxml text code + +# read in a template XML document that would pass the schema validation. +# for now, this is just template for the tes, and in the future, we could +# have the template XML generated by the latest XSD in the lib folder. + +import os +import sys +from six.moves import input + +try: + from lxml import etree, objectify +except ImportError: + print ("This script requires the lxml Python library to validate the XML scenario file.") + print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") + print("Exiting...") + sys.exit() + + +# ====================================================================================================================== + +# define the location of the template globally +# we reverse out the lib folder from the local tools module path + +tools_dir = os.path.split(os.path.realpath(__file__))[0] +ftot_program_directory = os.path.split(tools_dir)[0] +xml_template_file_location = os.path.join(ftot_program_directory, "lib", "v6_temp_Scenario.xml") + + + +def repl_old(): + + should_quit = False + + xml_file_location = "" + while not should_quit: + print ("XML Utility Options") + print ("-------------------") + print ("(1) ------ Create new template with defaults") + print ("(2) ------ Upgrade an old XML") + print ("(3) ------ Quit....") + ux_response = get_user_input("enter a selection") + if ux_response == "1": + print("XML task") + # (1) raw_input() on default XML + # output a fully commented XML template for the user to fill out manually + xml_file_location = generate_xml() + elif ux_response == "2": + # (2) upgrade xml + # compare an old XML file with a new tool + xml_file_location = xml_upgrade_tool() + elif ux_response == "3": + should_quit = True + else: + print("warning: response: {} -- not recognized. please enter a valid value: 1, 2, 3".format(ux_response)) + + return xml_file_location +# ====================================================================================================================== + + +def load_scenario_config_file(fullPathToXmlConfigFile): + + if not os.path.exists(fullPathToXmlConfigFile): + raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) + + if fullPathToXmlConfigFile.rfind(".xml") < 0: + raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) + + parser = etree.XMLParser(remove_blank_text=True) + return etree.parse(fullPathToXmlConfigFile, parser) + +# ============================================================================== + + +def do_the_upgrades(the_template_etree, the_old_xml_etree): + + # load both documents to the lxml etree + # iterate through the template + # -------------------------------------------------------------- + item_counter = 0 + for temp_elem in the_template_etree.getiterator(): + item_counter += 1 + + # check if the temp_element has the .find attribute + # if it does, then search for the index + # to '{' char at the end of namespace (e.g. {FTOT}Scenario) + # if there is no attribute, we continue b/c its probably a comment. + if not hasattr(temp_elem.tag, 'find'): + continue + clean_temp_elem = clean_element_name(temp_elem) + + # if the text in the element is just white space (ignore it) + # -------------------------------------------------------------- + if temp_elem.text == "": + continue + + # otherwise, inspect the element. + + # now we are going to iterate through the old XML and try to find a + # matching records. If we find one, we'll flip the set_flag variable + # (intialized below) to True, to indicate we found one. If we get to + # the end of the old XML loop and there is no match, we query the user. + # ---------------------------------------------------------------------- + set_flag = False + + for old_elem in the_old_xml_etree.getiterator(): + + # do a little bit of clean up on the namespaces... + + # first make sure its not a comment, + # by checking if it has the ".find" attribute + # ------------------------------------------- + if not hasattr(old_elem.tag, 'find'): continue # (1) + + # find the index of the end of the namespace + # then clean the name so there are no namespaces + # to mess with the compare + # ------------------------------------------- + clean_old_elem = clean_element_name(old_elem) + + # check the cleaned tags for a match + # ------------------------------------------- + + # a matched tag scenario + if clean_temp_elem == clean_old_elem: + # get the parent element name + temp_parent = temp_elem.getparent() + old_parent = old_elem.getparent() + # check if the parent Null + clean_temp_parent = None + clean_old_parent = None + if temp_parent is not None: + clean_temp_parent = clean_element_name(temp_parent) + if old_parent is not None: + clean_old_parent = clean_element_name(old_parent) + + if clean_temp_parent != clean_old_parent: + # if the parents don't match, keep iterating. + continue + + # check if there is a "default response" + # --------------------------------------- + ux_response = is_a_default_new_tag(clean_temp_elem) + if ux_response == "": + ux_script = "which one do you want keep? enter: old, new, edit, or quit" + ux_response = get_user_input(ux_script) + + # if not, ask the user which one to keep + # --------------------------------------- + if ux_response == "old": + temp_elem.text = old_elem.text + set_flag = True + elif ux_response == "new": + set_flag = True + elif ux_response == "edit": + temp_elem.text = get_user_input("enter a value press [return]") + set_flag = True + elif ux_response in ["quit", "exit"]: + io_error = "user asked to quit...quiting without saving" + raise IOError(io_error) + else: + print("warning: '{}' is not a valid input. try again... or quit".format(ux_response)) + + # a no match scenario. + # --------------------- + else: + continue + + # if at the end of the loop we didnt match an old element + # against the new template, get input from the user + # current default behavior is to keep the + # new template values as the default + if not set_flag: + + # accept the template element as the default value + # ------------------------------------------------- + temp_elem.text = temp_elem.text # this doesnt actually do anything + + # uncomment to allow the user to specify the new tag + # ---------------------------------------------------- + #ux_string = "The temp_element {} -- didn't match against the old xml file... it needs a value \n temp_xml value: {}".format(clean_temp_elem, temp_elem.text) + #temp_elem.text = get_user_input(ux_string) + + # record the change in the log + print("setting temp_elem: {} to {}".format(clean_temp_elem, temp_elem.text)) + + return + +# ============================================================================== + + +def record_deprecated_elements(): + pass + # now, check for deprecated elements by looping through the old document + # for each elemet in the old document + # check if there is a match in the new document. + # if there is not, then log it! + + +# ============================================================================== + + +def is_a_default_new_tag(clean_temp_elem): + old_or_new = "" + + default_new_list = [ + "Scenario_Schema_Version" + ] + default_old_list = [ + "Scenario_Name", + "Scenario_Description", + "Common_Data_Folder", + "Base_Network_Gdb", + "Base_RMP_Layer" + ] + + if clean_temp_elem in default_new_list: old_or_new = "new" + else: old_or_new = "old" + + return old_or_new + +# ============================================================================== + + +def clean_element_name(element): + # REMOVE THE NAMESPACE FROM THE FRONT OF THE TAG + # e.g. {FTOT}Scenario --> Scenario + # find the index of the end of the namespace + # then clean the name so there are no namespaces + # to mess with the compare + #------------------------------------------- + ns_i = element.tag.find("}") + return element.tag[ns_i+1:] + +# ============================================================================== + + +def get_user_input(ux_string): + # ask the user for a XML file please + print (ux_string) + user_input = input('--> ') + print("user_input: {}".format(user_input)) + return user_input + +# ============================================================================== + + +def save_the_new_xml_file(the_temp_etree): + report_file_dir = False + should_quit = False + while not should_quit: + report_file_dir = get_user_input("enter a path to save the scenario file") + + if os.path.exists(str(report_file_dir)):should_quit = True + if report_file_dir == "quit": should_quit = True + if report_file_dir == False: print("warning: report file dir {} not found. enter a new path or type 'quit'".format(report_file_dir)) + + report_file_name = get_user_input("name of the scenario file") + if report_file_name .rfind(".xml") < 0: + report_file_name += ".xml" + + report_file = os.path.join(report_file_dir, report_file_name) + + with open(report_file, 'wb') as wf: + print("writing the file: {} ".format(report_file)) + the_temp_etree.write(wf, pretty_print=True) + print("done writing xml file: {}".format(report_file)) + + return report_file + +# ============================================================================== + + +def generate_xml(): + # copies the template xml file with default values + # from the lib folder to a location the user specifies + + the_temp_etree = load_scenario_config_file(xml_template_file_location) + return save_the_new_xml_file(the_temp_etree) + + +# ============================================================================== + + +def xml_upgrade_tool(): + + CLI_UX_TEMP_XML_PATH = "give me the template please...drag and drop is fine here" + + the_temp_etree = load_scenario_config_file(xml_template_file_location) +# # ask the user for a XML file please +# print ("give me the template please...drag and drop is fine here") +# the_template_path = raw_input('--> ') +# print("USER INPUT: the_template_path: {}".format(the_template_path)) +# the_temp_etree = load_scenario_config_file(the_template_path) + + + # ask the user for a XML file please + print ("give me the XML to upgrade please...drag and drop is fine here") + the_old_xml_path = input('--> ') + print("USER INPUT: the_old_xml_path: {}".format(the_old_xml_path)) + the_old_xml_etree = load_scenario_config_file(the_old_xml_path) + + print("about to do the upgrade analysis") + do_the_upgrades(the_temp_etree, the_old_xml_etree) + + print("saving the xml template as a new file") + return save_the_new_xml_file(the_temp_etree) + +# ============================================================================== + + +def should_quit(): + print ("returning to the FTOT Tools REPL") + return False + + +# ============================================================================== + + +menuItems = [ + { "create new template with default values": generate_xml}, + { "upgrade an old XML": xml_upgrade_tool}, + { "exit": should_quit }, +] + +def repl(): + stay_in_repl = True + os.system('cls') + while stay_in_repl: + print ("XML Tools") + for item in menuItems: + print("[" + str(menuItems.index(item)) + "] " + list(item.keys())[0]) + choice = input(">> ") + try: + if int(choice) < 0 : raise ValueError + # Call the matching function + stay_in_repl = list(menuItems[int(choice)].values())[0]() + except (ValueError, IndexError): + pass diff --git a/program/tools/network_disruption_tool.py b/program/tools/network_disruption_tool.py index 5da7b01..0bf0929 100644 --- a/program/tools/network_disruption_tool.py +++ b/program/tools/network_disruption_tool.py @@ -1,271 +1,271 @@ -import arcpy -import os -import datetime -import csv - -# The following code takes a GIS-based raster dataset representing exposure data (such as a flood depth grid dataset -# from HAZUS or some other source) and determines the maximum exposure value for each network segment -# within a user-specified tolerance. This is then converted to a level of disruption (defined as link -# availability). The current version of the tool simply has a binary result (link is either fully exposed or not -# exposed). Segments with no exposure are not included in the output. This output can also be generated manually if it -# is easier to manually identify exposed segments. This tool only currently works with the rail and road modes - - -def run_network_disruption_tool(): - print("FTOT network disruption tool") - print("-------------------------------") - print("") - print("") - - if arcpy.CheckExtension("Spatial") == "Available": - print("Spatial Analyst is available. Network Disruption tool will run.") - else: - error = ("Unable to get ArcGIS Pro spatial analyst extension, which is required to run this tool") - raise Exception(error) - - network_disruption_prep() - - -def network_disruption_prep(): - - start_time = datetime.datetime.now() - print('\nStart at ' + str(start_time)) - - # SETUP - print('Prompting for configuration ...') - - # Get path to input network - network = get_network() - - road_y_n, rail_y_n = get_mode_list() - - mode_list = [] - if road_y_n == 'y': - mode_list.append("road") - if rail_y_n == 'y': - mode_list.append("rail") - - input_exposure_grid = get_input_exposure_data() - - input_exposure_grid_field = get_input_exposure_data_field() - - search_tolerance = get_search_tolerance() - - output_dir = get_output_dir() - - # Hard-coding this for now, if more options are added, this can be revisited - link_availability_approach = 'binary' - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - output_gdb = 'disruption_analysis.gdb' - full_path_to_output_gdb = os.path.join(output_dir, output_gdb) - - if arcpy.Exists(full_path_to_output_gdb): - arcpy.Delete_management(full_path_to_output_gdb) - print('Deleted existing ' + full_path_to_output_gdb) - arcpy.CreateFileGDB_management(output_dir, output_gdb) - - arcpy.env.workspace = full_path_to_output_gdb - - # MAIN - # --------------------------------------------------------------------------- - - txt_output_fields = ['mode', 'unique_link_id', 'link_availability', input_exposure_grid_field] - - # Export to txt file - csv_out = os.path.join(output_dir, "disruption.csv") - - with open(csv_out, "w", newline='') as f: - wr = csv.writer(f) - wr.writerow(txt_output_fields) - - for mode in mode_list: - - # Extract raster cells that overlap the modes of interest - print('Extracting exposure values that overlap network for mode: {} ...'.format(mode)) - arcpy.CheckOutExtension("Spatial") - output_extract_by_mask = arcpy.sa.ExtractByMask(input_exposure_grid, os.path.join(network, mode)) - output_extract_by_mask.save(mode + "_exposure_grid_extract") - arcpy.CheckInExtension("Spatial") - - # Export raster to point - print('Converting raster to point for mode: {} ...'.format(mode)) - arcpy.RasterToPoint_conversion(mode + "_exposure_grid_extract", os.path.join( - full_path_to_output_gdb, mode + "_exposure_grid_points"), input_exposure_grid_field) - - # Setup field mapping so that maximum exposure at each network segment is captured - fms = arcpy.FieldMappings() - - fm = arcpy.FieldMap() - fm.addInputField(mode + "_exposure_grid_points", "grid_code") - fm.mergeRule = 'Maximum' - - fms.addFieldMap(fm) - - # Spatial join to network, selecting highest exposure value for each network segment - print('Identifying maximum exposure value for each network segment for mode: {} ...'.format(mode)) - arcpy.SpatialJoin_analysis(os.path.join(network, mode), mode + "_exposure_grid_points", - mode + "_with_exposure", - "JOIN_ONE_TO_ONE", "KEEP_ALL", - fms, - "WITHIN_A_DISTANCE_GEODESIC", search_tolerance) - - # Add new field to store extent of exposure - print('Calculating exposure levels for mode: {} ...'.format(mode)) - - arcpy.AddField_management(mode + "_with_exposure", "link_availability", "Float") - arcpy.AddField_management(mode + "_with_exposure", "comments", "Text") - - arcpy.MakeFeatureLayer_management(mode + "_with_exposure", mode + "_with_exposure_lyr") - - # Convert NULLS to 0 first - arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "grid_code IS NULL") - arcpy.CalculateField_management(mode + "_with_exposure_lyr", "grid_code", 0, "PYTHON_9.3") - arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "CLEAR_SELECTION") - - if link_availability_approach == 'binary': - # 0 = full exposure/not traversable. 1 = no exposure/link fully available - arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "grid_code > 0") - arcpy.CalculateField_management(mode + "_with_exposure_lyr", "link_availability", 0, "PYTHON_9.3") - arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "SWITCH_SELECTION") - arcpy.CalculateField_management(mode + "_with_exposure_lyr", "link_availability", 1, "PYTHON_9.3") - - print('Finalizing outputs ... for mode: {} ...'.format(mode)) - - # Rename grid_code back to the original exposure grid field provided in raster dataset. - arcpy.AlterField_management(mode + "_with_exposure", 'grid_code', input_exposure_grid_field) - - fields = ['OBJECTID', 'link_availability', input_exposure_grid_field] - - # Only worry about disrupted links-- - # anything with a link availability of 1 is not disrupted and doesn't need to be included. - arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "link_availability <> 1") - - with open(csv_out, "a", newline='') as csv_file: - with arcpy.da.SearchCursor(mode + "_with_exposure_lyr", fields) as cursor: - for row in cursor: - converted_list = [str(element) for element in row] - joined_string = ",".join(converted_list) - csv_file.write("{},{}\n".format(mode, joined_string)) - - end_time = datetime.datetime.now() - total_run_time = end_time - start_time - print("\nEnd at {}. Total run time {}".format(end_time, total_run_time)) - -# ============================================================================== - - -def get_network(): - while True: - # Network Disruption Tool FTOT Network Path - print("network disruption tool | step 1/6:") - print("-------------------------------") - print("FTOT network path: ") - print("Determines the network to use for the disruption analysis") - network_gdb = input('----------------------> ') - print("USER INPUT: the FTOT network gdb path: {}".format(network_gdb)) - if not os.path.exists(network_gdb): - print("the following path is not valid. Please enter a valid path to an FTOT network gdb") - print("os.path.exists == False for: {}".format(network_gdb)) - continue - else: - # Valid value - break - return network_gdb - -# ============================================================================== - - -def get_mode_list(): - while True: - print("network disruption tool | step 2/6:") - print("-------------------------------") - print("apply disruption to road?: y or n") - # Only allowing this tool to work on road/rail for now - road_y_n = input('----------------------> ') - print("USER INPUT: disrupt road: {}".format(road_y_n)) - if road_y_n not in ["y", "n", "Y", "N"]: - print("must type y or n") - continue - else: - # Valid value - break - - while True: - print("apply disruption to rail?: y or n") - rail_y_n = input('----------------------> ') - print("USER INPUT: disrupt rail: {}".format(rail_y_n)) - if rail_y_n not in ["y", "n", "Y", "N"]: - print("must type y or n") - continue - else: - # Valid value - break - - return road_y_n, rail_y_n - - -# ============================================================================== - - -def get_input_exposure_data(): - while True: - # Network Disruption Tool Input Exposure Data - print("network disruption tool | step 3/6:") - print("-------------------------------") - print("FTOT gridded disruption data: ") - print("Determines the disruption data to be used for the disruption analysis (e.g. gridded flood exposure data)") - exposure_data = input('----------------------> ') - print("USER INPUT: the FTOT network exposure data: {}".format(exposure_data)) - if not arcpy.Exists(exposure_data): - print("the following path is not valid. Please enter a valid path to an FTOT disruption dataset") - print("arcpy.exists == False for: {}".format(exposure_data)) - else: - # Valid value - break - - return exposure_data - -# ============================================================================== - - -def get_input_exposure_data_field(): - # Network Disruption Tool Input Exposure Data Field - print("network disruption tool | step 4/6:") - print("-------------------------------") - print("Name of field which stores disruption data: ") - print("Typically this is Value but it may be something else") - exposure_data_field = input('----------------------> ') - print("USER INPUT: the FTOT network exposure data field: {}".format(exposure_data_field)) - - return exposure_data_field - -# ============================================================================== - - -def get_search_tolerance(): - # Network Disruption Tool Input Exposure Data Field - print("network disruption tool | step 5/6:") - print("-------------------------------") - print("Search Tolerance for determining exposure level of road segment-- a good default is 50% of the grid size") - print("Include units (e.g. meters, feet)") - search_tolerance = input('----------------------> ') - print("USER INPUT: the FTOT network search tolerance: {}".format(search_tolerance)) - - return search_tolerance - -# ============================================================================== - - -def get_output_dir(): - # Network Disruption Tool FTOT Network Path - print("network disruption tool | step 6/6:") - print("-------------------------------") - print("Output path: ") - print("Location where output data will be stored") - output_dir = input('----------------------> ') - print("USER INPUT: the output path: {}".format(output_dir)) - - return output_dir +import arcpy +import os +import datetime +import csv + +# The following code takes a GIS-based raster dataset representing exposure data (such as a flood depth grid dataset +# from HAZUS or some other source) and determines the maximum exposure value for each network segment +# within a user-specified tolerance. This is then converted to a level of disruption (defined as link +# availability). The current version of the tool simply has a binary result (link is either fully exposed or not +# exposed). Segments with no exposure are not included in the output. This output can also be generated manually if it +# is easier to manually identify exposed segments. This tool only currently works with the rail and road modes + + +def run_network_disruption_tool(): + print("FTOT network disruption tool") + print("-------------------------------") + print("") + print("") + + if arcpy.CheckExtension("Spatial") == "Available": + print("Spatial Analyst is available. Network Disruption tool will run.") + else: + error = ("Unable to get ArcGIS Pro spatial analyst extension, which is required to run this tool") + raise Exception(error) + + network_disruption_prep() + + +def network_disruption_prep(): + + start_time = datetime.datetime.now() + print('\nStart at ' + str(start_time)) + + # SETUP + print('Prompting for configuration ...') + + # Get path to input network + network = get_network() + + road_y_n, rail_y_n = get_mode_list() + + mode_list = [] + if road_y_n == 'y': + mode_list.append("road") + if rail_y_n == 'y': + mode_list.append("rail") + + input_exposure_grid = get_input_exposure_data() + + input_exposure_grid_field = get_input_exposure_data_field() + + search_tolerance = get_search_tolerance() + + output_dir = get_output_dir() + + # Hard-coding this for now, if more options are added, this can be revisited + link_availability_approach = 'binary' + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + output_gdb = 'disruption_analysis.gdb' + full_path_to_output_gdb = os.path.join(output_dir, output_gdb) + + if arcpy.Exists(full_path_to_output_gdb): + arcpy.Delete_management(full_path_to_output_gdb) + print('Deleted existing ' + full_path_to_output_gdb) + arcpy.CreateFileGDB_management(output_dir, output_gdb) + + arcpy.env.workspace = full_path_to_output_gdb + + # MAIN + # --------------------------------------------------------------------------- + + txt_output_fields = ['mode', 'unique_link_id', 'link_availability', input_exposure_grid_field] + + # Export to txt file + csv_out = os.path.join(output_dir, "disruption.csv") + + with open(csv_out, "w", newline='') as f: + wr = csv.writer(f) + wr.writerow(txt_output_fields) + + for mode in mode_list: + + # Extract raster cells that overlap the modes of interest + print('Extracting exposure values that overlap network for mode: {} ...'.format(mode)) + arcpy.CheckOutExtension("Spatial") + output_extract_by_mask = arcpy.sa.ExtractByMask(input_exposure_grid, os.path.join(network, mode)) + output_extract_by_mask.save(mode + "_exposure_grid_extract") + arcpy.CheckInExtension("Spatial") + + # Export raster to point + print('Converting raster to point for mode: {} ...'.format(mode)) + arcpy.RasterToPoint_conversion(mode + "_exposure_grid_extract", os.path.join( + full_path_to_output_gdb, mode + "_exposure_grid_points"), input_exposure_grid_field) + + # Setup field mapping so that maximum exposure at each network segment is captured + fms = arcpy.FieldMappings() + + fm = arcpy.FieldMap() + fm.addInputField(mode + "_exposure_grid_points", "grid_code") + fm.mergeRule = 'Maximum' + + fms.addFieldMap(fm) + + # Spatial join to network, selecting highest exposure value for each network segment + print('Identifying maximum exposure value for each network segment for mode: {} ...'.format(mode)) + arcpy.SpatialJoin_analysis(os.path.join(network, mode), mode + "_exposure_grid_points", + mode + "_with_exposure", + "JOIN_ONE_TO_ONE", "KEEP_ALL", + fms, + "WITHIN_A_DISTANCE_GEODESIC", search_tolerance) + + # Add new field to store extent of exposure + print('Calculating exposure levels for mode: {} ...'.format(mode)) + + arcpy.AddField_management(mode + "_with_exposure", "link_availability", "Float") + arcpy.AddField_management(mode + "_with_exposure", "comments", "Text") + + arcpy.MakeFeatureLayer_management(mode + "_with_exposure", mode + "_with_exposure_lyr") + + # Convert NULLS to 0 first + arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "grid_code IS NULL") + arcpy.CalculateField_management(mode + "_with_exposure_lyr", "grid_code", 0, "PYTHON_9.3") + arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "CLEAR_SELECTION") + + if link_availability_approach == 'binary': + # 0 = full exposure/not traversable. 1 = no exposure/link fully available + arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "grid_code > 0") + arcpy.CalculateField_management(mode + "_with_exposure_lyr", "link_availability", 0, "PYTHON_9.3") + arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "SWITCH_SELECTION") + arcpy.CalculateField_management(mode + "_with_exposure_lyr", "link_availability", 1, "PYTHON_9.3") + + print('Finalizing outputs ... for mode: {} ...'.format(mode)) + + # Rename grid_code back to the original exposure grid field provided in raster dataset. + arcpy.AlterField_management(mode + "_with_exposure", 'grid_code', input_exposure_grid_field) + + fields = ['OBJECTID', 'link_availability', input_exposure_grid_field] + + # Only worry about disrupted links-- + # anything with a link availability of 1 is not disrupted and doesn't need to be included. + arcpy.SelectLayerByAttribute_management(mode + "_with_exposure_lyr", "NEW_SELECTION", "link_availability <> 1") + + with open(csv_out, "a", newline='') as csv_file: + with arcpy.da.SearchCursor(mode + "_with_exposure_lyr", fields) as cursor: + for row in cursor: + converted_list = [str(element) for element in row] + joined_string = ",".join(converted_list) + csv_file.write("{},{}\n".format(mode, joined_string)) + + end_time = datetime.datetime.now() + total_run_time = end_time - start_time + print("\nEnd at {}. Total run time {}".format(end_time, total_run_time)) + +# ============================================================================== + + +def get_network(): + while True: + # Network Disruption Tool FTOT Network Path + print("network disruption tool | step 1/6:") + print("-------------------------------") + print("FTOT network path: ") + print("Determines the network to use for the disruption analysis") + network_gdb = input('----------------------> ') + print("USER INPUT: the FTOT network gdb path: {}".format(network_gdb)) + if not os.path.exists(network_gdb): + print("the following path is not valid. Please enter a valid path to an FTOT network gdb") + print("os.path.exists == False for: {}".format(network_gdb)) + continue + else: + # Valid value + break + return network_gdb + +# ============================================================================== + + +def get_mode_list(): + while True: + print("network disruption tool | step 2/6:") + print("-------------------------------") + print("apply disruption to road?: y or n") + # Only allowing this tool to work on road/rail for now + road_y_n = input('----------------------> ') + print("USER INPUT: disrupt road: {}".format(road_y_n)) + if road_y_n not in ["y", "n", "Y", "N"]: + print("must type y or n") + continue + else: + # Valid value + break + + while True: + print("apply disruption to rail?: y or n") + rail_y_n = input('----------------------> ') + print("USER INPUT: disrupt rail: {}".format(rail_y_n)) + if rail_y_n not in ["y", "n", "Y", "N"]: + print("must type y or n") + continue + else: + # Valid value + break + + return road_y_n, rail_y_n + + +# ============================================================================== + + +def get_input_exposure_data(): + while True: + # Network Disruption Tool Input Exposure Data + print("network disruption tool | step 3/6:") + print("-------------------------------") + print("FTOT gridded disruption data: ") + print("Determines the disruption data to be used for the disruption analysis (e.g. gridded flood exposure data)") + exposure_data = input('----------------------> ') + print("USER INPUT: the FTOT network exposure data: {}".format(exposure_data)) + if not arcpy.Exists(exposure_data): + print("the following path is not valid. Please enter a valid path to an FTOT disruption dataset") + print("arcpy.exists == False for: {}".format(exposure_data)) + else: + # Valid value + break + + return exposure_data + +# ============================================================================== + + +def get_input_exposure_data_field(): + # Network Disruption Tool Input Exposure Data Field + print("network disruption tool | step 4/6:") + print("-------------------------------") + print("Name of field which stores disruption data: ") + print("Typically this is Value but it may be something else") + exposure_data_field = input('----------------------> ') + print("USER INPUT: the FTOT network exposure data field: {}".format(exposure_data_field)) + + return exposure_data_field + +# ============================================================================== + + +def get_search_tolerance(): + # Network Disruption Tool Input Exposure Data Field + print("network disruption tool | step 5/6:") + print("-------------------------------") + print("Search Tolerance for determining exposure level of road segment-- a good default is 50% of the grid size") + print("Include units (e.g. meters, feet)") + search_tolerance = input('----------------------> ') + print("USER INPUT: the FTOT network search tolerance: {}".format(search_tolerance)) + + return search_tolerance + +# ============================================================================== + + +def get_output_dir(): + # Network Disruption Tool FTOT Network Path + print("network disruption tool | step 6/6:") + print("-------------------------------") + print("Output path: ") + print("Location where output data will be stored") + output_dir = input('----------------------> ') + print("USER INPUT: the output path: {}".format(output_dir)) + + return output_dir diff --git a/program/tools/odpair_util2.py b/program/tools/odpair_util2.py index 81242c8..51056fe 100644 --- a/program/tools/odpair_util2.py +++ b/program/tools/odpair_util2.py @@ -1,118 +1,118 @@ -# -*- coding: utf-8 -*- -""" -Created on Fri Jan 05 11:00:43 2018 - -@author: Matthew.Pearlson.CTR -""" - - -import os -import csv -import math -import datetime -import glob -import ntpath -#import ftot_supporting - -# create a big csv file to store all the entries -# batch, from location id, to location id - -# get OD pair csv files - -# iterate over the OD pair csvs -# and write each line into the new one - -def odpair_csv_util(): - - - print ("copy this file and run it from logs\multiprocessor_batch_solve dir to parse the batch logs. ") - #make a directory for the report to go into - log_dir = os.curdir - print (log_dir) - report_directory = os.path.join(log_dir, "report") - if not os.path.exists(report_directory): - os.makedirs(report_directory) - - # get all of the csv files matching the pattern - - log_files = glob.glob(os.path.join(log_dir, "*.csv")) - print ("len(log_files): {}".format(len(log_files))) - - log_file_list = [] - # add log file name and date to dictionary. each entry in the array - # will be a tuple of (log_file_name, datetime object) - - for log_file in log_files: - - path_to, the_file_name = ntpath.split(log_file) - - print (the_file_name) - - the_batch = the_file_name.split("_od_pairs") - - with open(log_file, 'r') as rf: - - for line in rf: - if line.find("O_Name"): - continue - elif line.find("D_Name"): - continue - else: - message = line. - - # do some sorting of the messages here - if message.find("Batch: ") > -1: - if message.find("- Total Runtime (HMS):") > -1: - pass - else: - batch_number = message.split(" - ")[0] - the_task = message.split(" - ")[1] - time_strings = message.split(" - ")[2] - start_time = time_strings.split("Runtime (HMS): ")[0].split("Start time: \t")[1].lstrip().rstrip() - run_time = time_strings.split("Runtime (HMS): ")[1].lstrip().rstrip() - message_dict["ALL_OPERATIONS"].append((the_batch, the_task, start_time, run_time)) - - if message.find("Total Runtime (HMS):") > -1: - - run_time = message.split("Total Runtime (HMS):")[1] - message_dict["TOTAL_TIME"].append((the_batch, time_stamp, run_time)) - - elif message.find("arcpy.na.Solve()") > -1: - run_time = message.split("Runtime (HMS): ")[1].lstrip().rstrip() - message_dict["ROUTE_SOLVE"].append((the_batch, time_stamp, run_time)) - - - message_dict["EVERYTHING"].append((the_batch, time_stamp, message)) - - # dump to files - # --------------- - - # total_runtime - #---------------- - - for message_type in message_dict: - report_file_name = 'report_{}_'.format(message_type) + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + ".csv" - report_file = os.path.join(report_directory, report_file_name) - - if message_type == "ALL_OPERATIONS": - with open(report_file, 'w') as wf: - wf.write('Batch, Opeartion, Start_Time, Run_Time\n') - for x in message_dict[message_type]: - wf.write('{}, {}, {}, {}\n'.format(x[0], x[1], x[2], x[3])) - else: - with open(report_file, 'w') as wf: - wf.write('Batch, Timestamp, Message\n') - for x in message_dict[message_type]: - wf.write('{}, {}, {}'.format(x[0], x[1], x[2])) - - print ("Report file location: {}".format(report_file)) - print ("all done.") - - - -if __name__ == "__main__": - - os.system('cls') - - odpair_csv_util() - +# -*- coding: utf-8 -*- +""" +Created on Fri Jan 05 11:00:43 2018 + +@author: Matthew.Pearlson.CTR +""" + + +import os +import csv +import math +import datetime +import glob +import ntpath +#import ftot_supporting + +# create a big csv file to store all the entries +# batch, from location id, to location id + +# get OD pair csv files + +# iterate over the OD pair csvs +# and write each line into the new one + +def odpair_csv_util(): + + + print ("copy this file and run it from logs\multiprocessor_batch_solve dir to parse the batch logs. ") + #make a directory for the report to go into + log_dir = os.curdir + print (log_dir) + report_directory = os.path.join(log_dir, "report") + if not os.path.exists(report_directory): + os.makedirs(report_directory) + + # get all of the csv files matching the pattern + + log_files = glob.glob(os.path.join(log_dir, "*.csv")) + print ("len(log_files): {}".format(len(log_files))) + + log_file_list = [] + # add log file name and date to dictionary. each entry in the array + # will be a tuple of (log_file_name, datetime object) + + for log_file in log_files: + + path_to, the_file_name = ntpath.split(log_file) + + print (the_file_name) + + the_batch = the_file_name.split("_od_pairs") + + with open(log_file, 'r') as rf: + + for line in rf: + if line.find("O_Name"): + continue + elif line.find("D_Name"): + continue + else: + message = line. + + # do some sorting of the messages here + if message.find("Batch: ") > -1: + if message.find("- Total Runtime (HMS):") > -1: + pass + else: + batch_number = message.split(" - ")[0] + the_task = message.split(" - ")[1] + time_strings = message.split(" - ")[2] + start_time = time_strings.split("Runtime (HMS): ")[0].split("Start time: \t")[1].lstrip().rstrip() + run_time = time_strings.split("Runtime (HMS): ")[1].lstrip().rstrip() + message_dict["ALL_OPERATIONS"].append((the_batch, the_task, start_time, run_time)) + + if message.find("Total Runtime (HMS):") > -1: + + run_time = message.split("Total Runtime (HMS):")[1] + message_dict["TOTAL_TIME"].append((the_batch, time_stamp, run_time)) + + elif message.find("arcpy.na.Solve()") > -1: + run_time = message.split("Runtime (HMS): ")[1].lstrip().rstrip() + message_dict["ROUTE_SOLVE"].append((the_batch, time_stamp, run_time)) + + + message_dict["EVERYTHING"].append((the_batch, time_stamp, message)) + + # dump to files + # --------------- + + # total_runtime + #---------------- + + for message_type in message_dict: + report_file_name = 'report_{}_'.format(message_type) + datetime.datetime.now().strftime("%Y_%m_%d_%H-%M-%S") + ".csv" + report_file = os.path.join(report_directory, report_file_name) + + if message_type == "ALL_OPERATIONS": + with open(report_file, 'w') as wf: + wf.write('Batch, Opeartion, Start_Time, Run_Time\n') + for x in message_dict[message_type]: + wf.write('{}, {}, {}, {}\n'.format(x[0], x[1], x[2], x[3])) + else: + with open(report_file, 'w') as wf: + wf.write('Batch, Timestamp, Message\n') + for x in message_dict[message_type]: + wf.write('{}, {}, {}'.format(x[0], x[1], x[2])) + + print ("Report file location: {}".format(report_file)) + print ("all done.") + + + +if __name__ == "__main__": + + os.system('cls') + + odpair_csv_util() + diff --git a/program/tools/odpairs_utils.py b/program/tools/odpairs_utils.py index 9df85fe..2063f33 100644 --- a/program/tools/odpairs_utils.py +++ b/program/tools/odpairs_utils.py @@ -1,71 +1,71 @@ -# -*- coding: utf-8 -*- -""" -Created on Fri Jan 05 11:09:51 2018 - -@author: Matthew.Pearlson.CTR -""" - -import csv -import sys -import os -import glob -import ntpath - - - -log_dir = os.curdir -print (log_dir) - -#make a directory for the report to go into -report_directory = os.path.join(log_dir, "report") -if not os.path.exists(report_directory): - os.makedirs(report_directory) - -report_file_name = "report_od_pairs.csv" -report_file = os.path.join(report_directory, report_file_name) - - -# get all of the csv files matching the pattern - -log_files = glob.glob(os.path.join(log_dir, "*.csv")) -print ("len(log_files): {}".format(len(log_files))) - -from_location_list = [] -to_location_list = [] -location_list = [] -# add log file name and date to dictionary. each entry in the array -# will be a tuple of (log_file_name, datetime object) - -# out locations that are showing up uniquely -out_string = "99_OUT,98_OUT,97_OUT,96_OUT,95_OUT,94_OUT,93_OUT,92_OUT,91_OUT,90_OUT,9_OUT,89_OUT,88_OUT,87_OUT,86_OUT,85_OUT,84_OUT,83_OUT,82_OUT,81_OUT,80_OUT,8_OUT,79_OUT,78_OUT,77_OUT,76_OUT,75_OUT,74_OUT,73_OUT,72_OUT,71_OUT,70_OUT,7_OUT,69_OUT,68_OUT,67_OUT,66_OUT,65_OUT,64_OUT,63_OUT,62_OUT,61_OUT,60_OUT,6_OUT,59_OUT,58_OUT,57_OUT,56_OUT,55_OUT,54_OUT,53_OUT,52_OUT,51_OUT,50_OUT,5_OUT,49_OUT,48_OUT,47_OUT,46_OUT,45_OUT,44_OUT,43_OUT,42_OUT,41_OUT,40_OUT,4_OUT,39_OUT,38_OUT,37_OUT,36_OUT,35_OUT,34_OUT,33_OUT,32_OUT,31_OUT,3056_OUT,3023_OUT,30_OUT,3_OUT,2957_OUT,2956_OUT,2955_OUT,2954_OUT,2950_OUT,2949_OUT,2938_OUT,2937_OUT,2936_OUT,2935_OUT,2934_OUT,2933_OUT,2932_OUT,2910_OUT,2909_OUT,2908_OUT,2907_OUT,2906_OUT,2905_OUT,2904_OUT,2902_OUT,2901_OUT,2900_OUT,29_OUT,2899_OUT,2898_OUT,2891_OUT,2890_OUT,2889_OUT,2888_OUT,2870_OUT,2869_OUT,2868_OUT,2867_OUT,2865_OUT,2864_OUT,2848_OUT,2847_OUT,2846_OUT,2845_OUT,2844_OUT,2843_OUT,2842_OUT,2841_OUT,2840_OUT,2839_OUT,2838_OUT,2837_OUT,2836_OUT,2835_OUT,2834_OUT,2833_OUT,2832_OUT,2831_OUT,2830_OUT,2829_OUT,2828_OUT,2827_OUT,2826_OUT,2825_OUT,2824_OUT,2823_OUT,2822_OUT,2821_OUT,2820_OUT,2819_OUT,2818_OUT,2817_OUT,2816_OUT,2815_OUT,2814_OUT,2813_OUT,2812_OUT,2811_OUT,2810_OUT,2809_OUT,2808_OUT,2807_OUT,2806_OUT,2805_OUT,2804_OUT,2803_OUT,2802_OUT,2801_OUT,2800_OUT,28_OUT,2799_OUT,2798_OUT,2797_OUT,2796_OUT,2795_OUT,2794_OUT,2793_OUT,2792_OUT,2791_OUT,2790_OUT,2789_OUT,2788_OUT,2787_OUT,2786_OUT,2785_OUT,2784_OUT,2783_OUT,2782_OUT,2781_OUT,2780_OUT,2779_OUT,2778_OUT,2777_OUT,2776_OUT,2775_OUT,2774_OUT,2773_OUT,2772_OUT,2771_OUT,2770_OUT,2769_OUT,2768_OUT,2767_OUT,2766_OUT,2765_OUT,2764_OUT,2763_OUT,2762_OUT,2761_OUT,2760_OUT,2759_OUT,2758_OUT,2757_OUT,2756_OUT,2755_OUT,2754_OUT,2753_OUT,2752_OUT,2751_OUT,2750_OUT,2749_OUT,2748_OUT,2747_OUT,2746_OUT,2745_OUT,2744_OUT,2743_OUT,2742_OUT,2741_OUT,2740_OUT,2739_OUT,2738_OUT,2737_OUT,2736_OUT,2735_OUT,2734_OUT,2733_OUT,2732_OUT,2731_OUT,2730_OUT,2729_OUT,2728_OUT,2727_OUT,2726_OUT,2725_OUT,2724_OUT,2723_OUT,2722_OUT,2721_OUT,2720_OUT,2719_OUT,2718_OUT,2717_OUT,2716_OUT,2715_OUT,2714_OUT,2713_OUT,2712_OUT,2711_OUT,2710_OUT,2709_OUT,2708_OUT,2707_OUT,27_OUT,26_OUT,25_OUT,24_OUT,2328_OUT,2327_OUT,232_OUT,2303_OUT,2302_OUT,2301_OUT,23_OUT,2299_OUT,2298_OUT,2296_OUT,2295_OUT,229_OUT,2288_OUT,2287_OUT,2286_OUT,228_OUT,227_OUT,2259_OUT,2258_OUT,2257_OUT,2256_OUT,2237_OUT,2236_OUT,2235_OUT,2203_OUT,2202_OUT,2201_OUT,2200_OUT,22_OUT,2196_OUT,2195_OUT,2185_OUT,2184_OUT,2183_OUT,2182_OUT,2181_OUT,2180_OUT,2179_OUT,2178_OUT,2177_OUT,2176_OUT,2175_OUT,21_OUT,205_OUT,204_OUT,203_OUT,202_OUT,201_OUT,200_OUT,20_OUT,2_OUT,199_OUT,197_OUT,196_OUT,195_OUT,1945_OUT,1944_OUT,1943_OUT,1940_OUT,194_OUT,1939_OUT,1938_OUT,1937_OUT,1930_OUT,193_OUT,1929_OUT,1928_OUT,1927_OUT,1926_OUT,1925_OUT,1924_OUT,1923_OUT,1922_OUT,1921_OUT,192_OUT,1916_OUT,1915_OUT,1914_OUT,1911_OUT,1910_OUT,1903_OUT,1902_OUT,19_OUT,1882_OUT,1881_OUT,1877_OUT,1876_OUT,1873_OUT,1872_OUT,1871_OUT,1861_OUT,1860_OUT,186_OUT,185_OUT,184_OUT,183_OUT,182_OUT,18_OUT,1736_OUT,1735_OUT,1734_OUT,1731_OUT,1730_OUT,1729_OUT,1728_OUT,1727_OUT,1726_OUT,1725_OUT,1724_OUT,1723_OUT,1722_OUT,1721_OUT,1720_OUT,1719_OUT,1718_OUT,1717_OUT,1716_OUT,1715_OUT,1714_OUT,1713_OUT,1712_OUT,1711_OUT,1710_OUT,1709_OUT,1708_OUT,1707_OUT,1706_OUT,1705_OUT,1704_OUT,1703_OUT,1702_OUT,1701_OUT,1700_OUT,17_OUT,1699_OUT,1698_OUT,1697_OUT,1696_OUT,1695_OUT,1694_OUT,1693_OUT,1692_OUT,1691_OUT,1690_OUT,1689_OUT,1688_OUT,1687_OUT,1686_OUT,1685_OUT,1684_OUT,1683_OUT,1682_OUT,1681_OUT,1680_OUT,1679_OUT,1678_OUT,1677_OUT,1676_OUT,1675_OUT,1674_OUT,1673_OUT,1672_OUT,1671_OUT,1670_OUT,1669_OUT,1668_OUT,1667_OUT,1666_OUT,1665_OUT,1664_OUT,1663_OUT,1662_OUT,1661_OUT,1660_OUT,1659_OUT,1658_OUT,1657_OUT,1656_OUT,1655_OUT,1654_OUT,1653_OUT,1652_OUT,1651_OUT,1650_OUT,165_OUT,1649_OUT,1648_OUT,1647_OUT,1646_OUT,1645_OUT,1644_OUT,1643_OUT,1642_OUT,1641_OUT,1640_OUT,164_OUT,1639_OUT,1638_OUT,1637_OUT,1636_OUT,1635_OUT,1634_OUT,1633_OUT,1632_OUT,1631_OUT,1630_OUT,,1620_OUT,162_OUT,1619_OUT,1618_OUT,1617_OUT,1616_OUT,1615_OUT,1614_OUT,1613_OUT,1612_OUT,1611_OUT,1610_OUT,1609_OUT,1608_OUT,1607_OUT,1606_OUT,1605_OUT,1604_OUT,1603_OUT,1602_OUT,1601_OUT,1600_OUT,160_OUT,16_OUT,1599_OUT,1598_OUT,1597_OUT,1596_OUT,1595_OUT,1594_OUT,1593_OUT,1592_OUT,1591_OUT,1590_OUT,159_OUT,1589_OUT,1588_OUT,1587_OUT,1586_OUT,1585_OUT,1584_OUT,1583_OUT,1582_OUT,1581_OUT,1580_OUT,158_OUT,1579_OUT,1578_OUT,1577_OUT,1576_OUT,1575_OUT,1574_OUT,1573_OUT,1572_OUT,1571_OUT,1570_OUT,1569_OUT,1568_OUT,1567_OUT,1566_OUT,1565_OUT,1564_OUT,1563_OUT,1562_OUT,1561_OUT,1560_OUT,1559_OUT,1558_OUT,1557_OUT,1556_OUT,1555_OUT,1554_OUT,1553_OUT,1552_OUT,1551_OUT,1550_OUT,1549_OUT,1548_OUT,1547_OUT,1546_OUT,1545_OUT,1544_OUT,1543_OUT,1542_OUT,1541_OUT,1540_OUT,1539_OUT,1538_OUT,1537_OUT,1536_OUT,1535_OUT,1534_OUT,1533_OUT,1532_OUT,1531_OUT,1530_OUT,1529_OUT,1528_OUT,1527_OUT,1526_OUT,1525_OUT,1524_OUT,1523_OUT,1522_OUT,1521_OUT,1520_OUT,1519_OUT,1518_OUT,1517_OUT,1516_OUT,1515_OUT,1514_OUT,1513_OUT,1512_OUT,1511_OUT,1510_OUT,1509_OUT,1508_OUT,1507_OUT,1506_OUT,1505_OUT,1504_OUT,1503_OUT,1502_OUT,1501_OUT,1500_OUT,15_OUT,1499_OUT,1498_OUT,1497_OUT,1496_OUT,1495_OUT,1494_OUT,1493_OUT,1492_OUT,1491_OUT,1490_OUT,1489_OUT,1488_OUT,1468_OUT,1467_OUT,1466_OUT,1465_OUT,1464_OUT,1463_OUT,1462_OUT,1461_OUT,1460_OUT,1459_OUT,142_OUT,141_OUT,140_OUT,14_OUT,139_OUT,138_OUT,137_OUT,136_OUT,135_OUT,134_OUT,133_OUT,132_OUT,131_OUT,130_OUT,13_OUT,129_OUT,128_OUT,127_OUT,126_OUT,125_OUT,124_OUT,123_OUT,122_OUT,121_OUT,120_OUT,12_OUT,119_OUT,118_OUT,117_OUT,116_OUT,115_OUT,114_OUT,113_OUT,112_OUT,111_OUT,110_OUT,11_OUT,109_OUT,108_OUT,107_OUT,106_OUT,105_OUT,104_OUT,103_OUT,102_OUT,101_OUT,100_OUT,10_OUT,1_OUT,163_OUT,1629_OUT,1628_OUT,1627_OUT,1626_OUT,1625_OUT,1624_OUT,1623_OUT,1622_OUT,1621_OUT" -out_string_parsed = out_string.replace("_OUT","") -out_list = out_string_parsed.split(',') -print (len(out_list)) - - - -with open(report_file, 'w') as wf: - for log_file in log_files: - from_location_list = [] - to_location_list = [] - path_to, the_file_name = ntpath.split(log_file) - f_in = open(log_file, 'rb') - try: - reader = csv.reader(f_in) - for row in reader: - from_location = row[0].replace("_OUT", "") - to_location = row[1].replace("_IN", "") -# from_location = row[0] -# to_location = row[1] - - - if from_location in out_list and from_location not in from_location_list: - wf.write(str(the_file_name+","+from_location+'\n')) - from_location_list.append(from_location) - - if to_location in out_list and to_location not in to_location_list: - wf.write(str(the_file_name+","+to_location+'\n')) - to_location_list.append(to_location) - - finally: - f_in.close() +# -*- coding: utf-8 -*- +""" +Created on Fri Jan 05 11:09:51 2018 + +@author: Matthew.Pearlson.CTR +""" + +import csv +import sys +import os +import glob +import ntpath + + + +log_dir = os.curdir +print (log_dir) + +#make a directory for the report to go into +report_directory = os.path.join(log_dir, "report") +if not os.path.exists(report_directory): + os.makedirs(report_directory) + +report_file_name = "report_od_pairs.csv" +report_file = os.path.join(report_directory, report_file_name) + + +# get all of the csv files matching the pattern + +log_files = glob.glob(os.path.join(log_dir, "*.csv")) +print ("len(log_files): {}".format(len(log_files))) + +from_location_list = [] +to_location_list = [] +location_list = [] +# add log file name and date to dictionary. each entry in the array +# will be a tuple of (log_file_name, datetime object) + +# out locations that are showing up uniquely +out_string = "99_OUT,98_OUT,97_OUT,96_OUT,95_OUT,94_OUT,93_OUT,92_OUT,91_OUT,90_OUT,9_OUT,89_OUT,88_OUT,87_OUT,86_OUT,85_OUT,84_OUT,83_OUT,82_OUT,81_OUT,80_OUT,8_OUT,79_OUT,78_OUT,77_OUT,76_OUT,75_OUT,74_OUT,73_OUT,72_OUT,71_OUT,70_OUT,7_OUT,69_OUT,68_OUT,67_OUT,66_OUT,65_OUT,64_OUT,63_OUT,62_OUT,61_OUT,60_OUT,6_OUT,59_OUT,58_OUT,57_OUT,56_OUT,55_OUT,54_OUT,53_OUT,52_OUT,51_OUT,50_OUT,5_OUT,49_OUT,48_OUT,47_OUT,46_OUT,45_OUT,44_OUT,43_OUT,42_OUT,41_OUT,40_OUT,4_OUT,39_OUT,38_OUT,37_OUT,36_OUT,35_OUT,34_OUT,33_OUT,32_OUT,31_OUT,3056_OUT,3023_OUT,30_OUT,3_OUT,2957_OUT,2956_OUT,2955_OUT,2954_OUT,2950_OUT,2949_OUT,2938_OUT,2937_OUT,2936_OUT,2935_OUT,2934_OUT,2933_OUT,2932_OUT,2910_OUT,2909_OUT,2908_OUT,2907_OUT,2906_OUT,2905_OUT,2904_OUT,2902_OUT,2901_OUT,2900_OUT,29_OUT,2899_OUT,2898_OUT,2891_OUT,2890_OUT,2889_OUT,2888_OUT,2870_OUT,2869_OUT,2868_OUT,2867_OUT,2865_OUT,2864_OUT,2848_OUT,2847_OUT,2846_OUT,2845_OUT,2844_OUT,2843_OUT,2842_OUT,2841_OUT,2840_OUT,2839_OUT,2838_OUT,2837_OUT,2836_OUT,2835_OUT,2834_OUT,2833_OUT,2832_OUT,2831_OUT,2830_OUT,2829_OUT,2828_OUT,2827_OUT,2826_OUT,2825_OUT,2824_OUT,2823_OUT,2822_OUT,2821_OUT,2820_OUT,2819_OUT,2818_OUT,2817_OUT,2816_OUT,2815_OUT,2814_OUT,2813_OUT,2812_OUT,2811_OUT,2810_OUT,2809_OUT,2808_OUT,2807_OUT,2806_OUT,2805_OUT,2804_OUT,2803_OUT,2802_OUT,2801_OUT,2800_OUT,28_OUT,2799_OUT,2798_OUT,2797_OUT,2796_OUT,2795_OUT,2794_OUT,2793_OUT,2792_OUT,2791_OUT,2790_OUT,2789_OUT,2788_OUT,2787_OUT,2786_OUT,2785_OUT,2784_OUT,2783_OUT,2782_OUT,2781_OUT,2780_OUT,2779_OUT,2778_OUT,2777_OUT,2776_OUT,2775_OUT,2774_OUT,2773_OUT,2772_OUT,2771_OUT,2770_OUT,2769_OUT,2768_OUT,2767_OUT,2766_OUT,2765_OUT,2764_OUT,2763_OUT,2762_OUT,2761_OUT,2760_OUT,2759_OUT,2758_OUT,2757_OUT,2756_OUT,2755_OUT,2754_OUT,2753_OUT,2752_OUT,2751_OUT,2750_OUT,2749_OUT,2748_OUT,2747_OUT,2746_OUT,2745_OUT,2744_OUT,2743_OUT,2742_OUT,2741_OUT,2740_OUT,2739_OUT,2738_OUT,2737_OUT,2736_OUT,2735_OUT,2734_OUT,2733_OUT,2732_OUT,2731_OUT,2730_OUT,2729_OUT,2728_OUT,2727_OUT,2726_OUT,2725_OUT,2724_OUT,2723_OUT,2722_OUT,2721_OUT,2720_OUT,2719_OUT,2718_OUT,2717_OUT,2716_OUT,2715_OUT,2714_OUT,2713_OUT,2712_OUT,2711_OUT,2710_OUT,2709_OUT,2708_OUT,2707_OUT,27_OUT,26_OUT,25_OUT,24_OUT,2328_OUT,2327_OUT,232_OUT,2303_OUT,2302_OUT,2301_OUT,23_OUT,2299_OUT,2298_OUT,2296_OUT,2295_OUT,229_OUT,2288_OUT,2287_OUT,2286_OUT,228_OUT,227_OUT,2259_OUT,2258_OUT,2257_OUT,2256_OUT,2237_OUT,2236_OUT,2235_OUT,2203_OUT,2202_OUT,2201_OUT,2200_OUT,22_OUT,2196_OUT,2195_OUT,2185_OUT,2184_OUT,2183_OUT,2182_OUT,2181_OUT,2180_OUT,2179_OUT,2178_OUT,2177_OUT,2176_OUT,2175_OUT,21_OUT,205_OUT,204_OUT,203_OUT,202_OUT,201_OUT,200_OUT,20_OUT,2_OUT,199_OUT,197_OUT,196_OUT,195_OUT,1945_OUT,1944_OUT,1943_OUT,1940_OUT,194_OUT,1939_OUT,1938_OUT,1937_OUT,1930_OUT,193_OUT,1929_OUT,1928_OUT,1927_OUT,1926_OUT,1925_OUT,1924_OUT,1923_OUT,1922_OUT,1921_OUT,192_OUT,1916_OUT,1915_OUT,1914_OUT,1911_OUT,1910_OUT,1903_OUT,1902_OUT,19_OUT,1882_OUT,1881_OUT,1877_OUT,1876_OUT,1873_OUT,1872_OUT,1871_OUT,1861_OUT,1860_OUT,186_OUT,185_OUT,184_OUT,183_OUT,182_OUT,18_OUT,1736_OUT,1735_OUT,1734_OUT,1731_OUT,1730_OUT,1729_OUT,1728_OUT,1727_OUT,1726_OUT,1725_OUT,1724_OUT,1723_OUT,1722_OUT,1721_OUT,1720_OUT,1719_OUT,1718_OUT,1717_OUT,1716_OUT,1715_OUT,1714_OUT,1713_OUT,1712_OUT,1711_OUT,1710_OUT,1709_OUT,1708_OUT,1707_OUT,1706_OUT,1705_OUT,1704_OUT,1703_OUT,1702_OUT,1701_OUT,1700_OUT,17_OUT,1699_OUT,1698_OUT,1697_OUT,1696_OUT,1695_OUT,1694_OUT,1693_OUT,1692_OUT,1691_OUT,1690_OUT,1689_OUT,1688_OUT,1687_OUT,1686_OUT,1685_OUT,1684_OUT,1683_OUT,1682_OUT,1681_OUT,1680_OUT,1679_OUT,1678_OUT,1677_OUT,1676_OUT,1675_OUT,1674_OUT,1673_OUT,1672_OUT,1671_OUT,1670_OUT,1669_OUT,1668_OUT,1667_OUT,1666_OUT,1665_OUT,1664_OUT,1663_OUT,1662_OUT,1661_OUT,1660_OUT,1659_OUT,1658_OUT,1657_OUT,1656_OUT,1655_OUT,1654_OUT,1653_OUT,1652_OUT,1651_OUT,1650_OUT,165_OUT,1649_OUT,1648_OUT,1647_OUT,1646_OUT,1645_OUT,1644_OUT,1643_OUT,1642_OUT,1641_OUT,1640_OUT,164_OUT,1639_OUT,1638_OUT,1637_OUT,1636_OUT,1635_OUT,1634_OUT,1633_OUT,1632_OUT,1631_OUT,1630_OUT,,1620_OUT,162_OUT,1619_OUT,1618_OUT,1617_OUT,1616_OUT,1615_OUT,1614_OUT,1613_OUT,1612_OUT,1611_OUT,1610_OUT,1609_OUT,1608_OUT,1607_OUT,1606_OUT,1605_OUT,1604_OUT,1603_OUT,1602_OUT,1601_OUT,1600_OUT,160_OUT,16_OUT,1599_OUT,1598_OUT,1597_OUT,1596_OUT,1595_OUT,1594_OUT,1593_OUT,1592_OUT,1591_OUT,1590_OUT,159_OUT,1589_OUT,1588_OUT,1587_OUT,1586_OUT,1585_OUT,1584_OUT,1583_OUT,1582_OUT,1581_OUT,1580_OUT,158_OUT,1579_OUT,1578_OUT,1577_OUT,1576_OUT,1575_OUT,1574_OUT,1573_OUT,1572_OUT,1571_OUT,1570_OUT,1569_OUT,1568_OUT,1567_OUT,1566_OUT,1565_OUT,1564_OUT,1563_OUT,1562_OUT,1561_OUT,1560_OUT,1559_OUT,1558_OUT,1557_OUT,1556_OUT,1555_OUT,1554_OUT,1553_OUT,1552_OUT,1551_OUT,1550_OUT,1549_OUT,1548_OUT,1547_OUT,1546_OUT,1545_OUT,1544_OUT,1543_OUT,1542_OUT,1541_OUT,1540_OUT,1539_OUT,1538_OUT,1537_OUT,1536_OUT,1535_OUT,1534_OUT,1533_OUT,1532_OUT,1531_OUT,1530_OUT,1529_OUT,1528_OUT,1527_OUT,1526_OUT,1525_OUT,1524_OUT,1523_OUT,1522_OUT,1521_OUT,1520_OUT,1519_OUT,1518_OUT,1517_OUT,1516_OUT,1515_OUT,1514_OUT,1513_OUT,1512_OUT,1511_OUT,1510_OUT,1509_OUT,1508_OUT,1507_OUT,1506_OUT,1505_OUT,1504_OUT,1503_OUT,1502_OUT,1501_OUT,1500_OUT,15_OUT,1499_OUT,1498_OUT,1497_OUT,1496_OUT,1495_OUT,1494_OUT,1493_OUT,1492_OUT,1491_OUT,1490_OUT,1489_OUT,1488_OUT,1468_OUT,1467_OUT,1466_OUT,1465_OUT,1464_OUT,1463_OUT,1462_OUT,1461_OUT,1460_OUT,1459_OUT,142_OUT,141_OUT,140_OUT,14_OUT,139_OUT,138_OUT,137_OUT,136_OUT,135_OUT,134_OUT,133_OUT,132_OUT,131_OUT,130_OUT,13_OUT,129_OUT,128_OUT,127_OUT,126_OUT,125_OUT,124_OUT,123_OUT,122_OUT,121_OUT,120_OUT,12_OUT,119_OUT,118_OUT,117_OUT,116_OUT,115_OUT,114_OUT,113_OUT,112_OUT,111_OUT,110_OUT,11_OUT,109_OUT,108_OUT,107_OUT,106_OUT,105_OUT,104_OUT,103_OUT,102_OUT,101_OUT,100_OUT,10_OUT,1_OUT,163_OUT,1629_OUT,1628_OUT,1627_OUT,1626_OUT,1625_OUT,1624_OUT,1623_OUT,1622_OUT,1621_OUT" +out_string_parsed = out_string.replace("_OUT","") +out_list = out_string_parsed.split(',') +print (len(out_list)) + + + +with open(report_file, 'w') as wf: + for log_file in log_files: + from_location_list = [] + to_location_list = [] + path_to, the_file_name = ntpath.split(log_file) + f_in = open(log_file, 'rb') + try: + reader = csv.reader(f_in) + for row in reader: + from_location = row[0].replace("_OUT", "") + to_location = row[1].replace("_IN", "") +# from_location = row[0] +# to_location = row[1] + + + if from_location in out_list and from_location not in from_location_list: + wf.write(str(the_file_name+","+from_location+'\n')) + from_location_list.append(from_location) + + if to_location in out_list and to_location not in to_location_list: + wf.write(str(the_file_name+","+to_location+'\n')) + to_location_list.append(to_location) + + finally: + f_in.close() diff --git a/program/tools/run_upgrade_tool.py b/program/tools/run_upgrade_tool.py index bbc7982..aaf4ed0 100644 --- a/program/tools/run_upgrade_tool.py +++ b/program/tools/run_upgrade_tool.py @@ -1,206 +1,206 @@ -# run.bat upgrade tool - -# the run.bat upgrade tool is used to generate run.bat files for the user. -# it is a command-line-interface (CLI) tool that prompts the user for the following information: -# - python installation (32 or 64 bit) -# - FTOT program directory -# - Scenario XML File -# - Candidate generation? Y/N -# - Output file location -# ============================================================================== -import os -from six.moves import input - -# ============================================================================== - - -def run_bat_upgrade_tool(): - os.system('CLS') - print("FTOT run.bat configuration tool") - print("-------------------------------") - print("") - print("") - - config_params = get_user_configs() - - save_the_new_run_bat_file(config_params) -# ============================================================================== - - -def get_user_configs(): - - print("run.bat configuration tool") - - # python environment - python = "C:\\FTOT\\python3_env\\python.exe" - print("setting python path: {}".format(python)) - while not os.path.exists(python): - python = input('----------------------> ') - print("USER INPUT: the python path: {}".format(python)) - if not os.path.exists(python): - print("warning: the following path is not valid. Please enter a valid path to python.exe") - print("warning: os.path.exists == False for: {}".format(python)) - - # FTOT program directory - ftot = "C:\\FTOT\\program\\ftot.py" # ftot.py location - print("setting ftot path: {}".format(ftot)) - while not os.path.exists(ftot): - ftot = input('----------------------> ') - print("USER INPUT: the ftot.py path: {}".format(ftot)) - if not os.path.exists(ftot): - print("warning: the following path is not valid. Please enter a valid path to ftot.py") - print("warning: os.path.exists == False for: {}".format(ftot)) - - # Scenario XML File - print("-------------------------------") - print("scenario.xml file path: (drag and drop is fine here)") - scenario_xml= "" - while not os.path.exists(scenario_xml): - scenario_xml = input('----------------------> ') - print("USER INPUT: the scenario.xml path: {}".format(scenario_xml)) - if not os.path.exists(scenario_xml): - print("warning: the following path is not valid. Please enter a valid path to scenario_xml") - print("warning: os.path.exists == False for: {}".format(scenario_xml)) - - # Candidate generation - print("-------------------------------") - print("include candidate generation steps)") - print("(1) YES") - print("(2) NO") - candidate_bool= "" - quit = False - while not quit: - candidate_bool = input('----------------------> ') - print("USER INPUT: candidate_bool: {}".format(candidate_bool)) - if candidate_bool == "1": - print("Candidate generation will be INCLUDED in the run configuration.") - candidate_bool = True - quit = True - elif candidate_bool == "2": - print("Candidate generation will be EXCLUDED in the run configuration.") - candidate_bool = False - quit = True - else: - print("warning: user response: {} is not recognized. Please enter a valid option (1 or 2).".format(candidate_bool)) - quit = False - - # Output file location - print("--------------------------------------") - print("location for the new .bat file: (full folder path)") - output_dir = "" - while not os.path.exists(output_dir): - output_dir = input('----------------------> ') - print("USER INPUT: the folder path: {}".format(output_dir)) - if not os.path.exists(output_dir): - print("warning: the following path is not valid. Please enter a valid path to output_dir") - print("warning: os.path.exists == False for: {}".format(output_dir)) - - return [python, ftot, scenario_xml, candidate_bool, output_dir] -# ======================================================================================================================= - - -def save_the_new_run_bat_file(config_params): - - # unpack the config parameters - python, ftot, scenario_xml, candidate_bool, output_dir = config_params - - run_bat_file = os.path.join(output_dir, "run_v6_1.bat") - with open(run_bat_file, 'w') as wf: - print("writing the file: {} ".format(run_bat_file)) - - # HEADER INFORMATION - header_info = """ - @ECHO OFF - cls - set PYTHONDONTWRITEBYTECODE=1 - REM default is #ECHO OFF, cls (clear screen), and disable .pyc files - REM for debugging REM @ECHO OFF line above to see commands - REM ------------------------------------------------- - - REM run_v6_1.bat generated by FTOT run.bat upgrade tool - REM ================================================= - - - REM ============================================== - REM ======== ENVIRONMENT VARIABLES =============== - REM ============================================== - set PYTHON="{}" - set FTOT="{}" - set XMLSCENARIO="{}" - """.format(python, ftot, scenario_xml) - wf.writelines(header_info.replace(" ", "")) # remove the indentation white space - - ftot_script_chunk_a = """ - - REM ============================================== - REM ======== RUN THE FTOT SCRIPT ================= - REM ============================================== - - REM SETUP: SETUP FTOT FOR THE SCENARIO - %PYTHON% %FTOT% %XMLSCENARIO% s || exit /b - - REM FACILITIES: ADD FACILITIES LOCATIONS AND - REM COMMODITY DATA FILES TO THE SCENARIO - %PYTHON% %FTOT% %XMLSCENARIO% f || exit /b - - REM CONNECTIVITY: CONNECT THE FACILITIES TO THE NETWORK AND - REM EXPORT THE FROM GIS TO NETWORKX MODULE - %PYTHON% %FTOT% %XMLSCENARIO% c || exit /b - - REM GRAPH: CREATE THE NETWORKX GRAPHS FROM THE - REM NETWORK AND FACILTIIES - %PYTHON% %FTOT% %XMLSCENARIO% g || exit /b - """ - - wf.writelines(ftot_script_chunk_a.replace(" ", "")) # remove the indentation white space - - if candidate_bool == True: - ftot_script_chunk_b = """ - REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION SETUP DATABASE - %PYTHON% %FTOT% %XMLSCENARIO% oc1 || exit /b - - REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION DEFINE & SOLVE PROBLEM - %PYTHON% %FTOT% %XMLSCENARIO% oc2 || exit /b - - REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION POSTPROCESS - %PYTHON% %FTOT% %XMLSCENARIO% oc3 || exit /b - - REM FACILITIES 2 : ADD FACILITIES LOCATIONS AND - REM COMMODITY DATA FILES TO THE SCENARIO - %PYTHON% %FTOT% %XMLSCENARIO% f2 || exit /b - - REM CONNECTIVITY 2: CONNECT THE FACILITIES TO THE NETWORK AND - REM EXPORT THE FROM GIS TO NETWORKX MODULE - %PYTHON% %FTOT% %XMLSCENARIO% c2 || exit /b - - REM GRAPH 2 : CREATE THE NETWORKX GRAPHS FROM THE - REM NETWORK AND FACILTIIES - %PYTHON% %FTOT% %XMLSCENARIO% g2 || exit /b - """ - wf.writelines(ftot_script_chunk_b.replace(" ", "")) # remove the indentation white space - - - ftot_script_chunk_c = """ - REM STEP OPTIMIZATION: SET UP THE OPTIMIZATION PROBLEM - %PYTHON% %FTOT% %XMLSCENARIO% o1 || exit /b - - REM STEP OPTIMIZATION: BUILD THE OPTIMIZATION PROBLEM AND SOLVE - %PYTHON% %FTOT% %XMLSCENARIO% o2 || exit /b - - REM POST-PROCESSING: POST PROCESS OPTIMAL RESULTS AND PREPARE REPORTING - %PYTHON% %FTOT% %XMLSCENARIO% p || exit /b - - REM REPORT: CREATE REPORTS OF THE CONFIGURATION, RESULTS, AND - REM WARNINGS FROM THE RUN - %PYTHON% %FTOT% %XMLSCENARIO% d || exit /b - - REM MAPS: GENERATE MAPS OF THE SCENARIO FOR EACH STEP - %PYTHON% %FTOT% %XMLSCENARIO% m || exit /b - - REM OPTIONAL MAPPING STEP FOR TIME AND COMMODITY DISTANCES - REM %PYTHON% %FTOT% %XMLSCENARIO% m2 || exit /b - """ - wf.writelines(ftot_script_chunk_c.replace(" ", "")) # remove the indentation white space - - print("file location: {}".format(run_bat_file)) +# run.bat upgrade tool + +# the run.bat upgrade tool is used to generate run.bat files for the user. +# it is a command-line-interface (CLI) tool that prompts the user for the following information: +# - python installation (32 or 64 bit) +# - FTOT program directory +# - Scenario XML File +# - Candidate generation? Y/N +# - Output file location +# ============================================================================== +import os +from six.moves import input + +# ============================================================================== + + +def run_bat_upgrade_tool(): + os.system('CLS') + print("FTOT run.bat configuration tool") + print("-------------------------------") + print("") + print("") + + config_params = get_user_configs() + + save_the_new_run_bat_file(config_params) +# ============================================================================== + + +def get_user_configs(): + + print("run.bat configuration tool") + + # python environment + python = "C:\\FTOT\\python3_env\\python.exe" + print("setting python path: {}".format(python)) + while not os.path.exists(python): + python = input('----------------------> ') + print("USER INPUT: the python path: {}".format(python)) + if not os.path.exists(python): + print("warning: the following path is not valid. Please enter a valid path to python.exe") + print("warning: os.path.exists == False for: {}".format(python)) + + # FTOT program directory + ftot = "C:\\FTOT\\program\\ftot.py" # ftot.py location + print("setting ftot path: {}".format(ftot)) + while not os.path.exists(ftot): + ftot = input('----------------------> ') + print("USER INPUT: the ftot.py path: {}".format(ftot)) + if not os.path.exists(ftot): + print("warning: the following path is not valid. Please enter a valid path to ftot.py") + print("warning: os.path.exists == False for: {}".format(ftot)) + + # Scenario XML File + print("-------------------------------") + print("scenario.xml file path: (drag and drop is fine here)") + scenario_xml= "" + while not os.path.exists(scenario_xml): + scenario_xml = input('----------------------> ') + print("USER INPUT: the scenario.xml path: {}".format(scenario_xml)) + if not os.path.exists(scenario_xml): + print("warning: the following path is not valid. Please enter a valid path to scenario_xml") + print("warning: os.path.exists == False for: {}".format(scenario_xml)) + + # Candidate generation + print("-------------------------------") + print("include candidate generation steps)") + print("(1) YES") + print("(2) NO") + candidate_bool= "" + quit = False + while not quit: + candidate_bool = input('----------------------> ') + print("USER INPUT: candidate_bool: {}".format(candidate_bool)) + if candidate_bool == "1": + print("Candidate generation will be INCLUDED in the run configuration.") + candidate_bool = True + quit = True + elif candidate_bool == "2": + print("Candidate generation will be EXCLUDED in the run configuration.") + candidate_bool = False + quit = True + else: + print("warning: user response: {} is not recognized. Please enter a valid option (1 or 2).".format(candidate_bool)) + quit = False + + # Output file location + print("--------------------------------------") + print("location for the new .bat file: (full folder path)") + output_dir = "" + while not os.path.exists(output_dir): + output_dir = input('----------------------> ') + print("USER INPUT: the folder path: {}".format(output_dir)) + if not os.path.exists(output_dir): + print("warning: the following path is not valid. Please enter a valid path to output_dir") + print("warning: os.path.exists == False for: {}".format(output_dir)) + + return [python, ftot, scenario_xml, candidate_bool, output_dir] +# ======================================================================================================================= + + +def save_the_new_run_bat_file(config_params): + + # unpack the config parameters + python, ftot, scenario_xml, candidate_bool, output_dir = config_params + + run_bat_file = os.path.join(output_dir, "run_v6_1.bat") + with open(run_bat_file, 'w') as wf: + print("writing the file: {} ".format(run_bat_file)) + + # HEADER INFORMATION + header_info = """ + @ECHO OFF + cls + set PYTHONDONTWRITEBYTECODE=1 + REM default is #ECHO OFF, cls (clear screen), and disable .pyc files + REM for debugging REM @ECHO OFF line above to see commands + REM ------------------------------------------------- + + REM run_v6_1.bat generated by FTOT run.bat upgrade tool + REM ================================================= + + + REM ============================================== + REM ======== ENVIRONMENT VARIABLES =============== + REM ============================================== + set PYTHON="{}" + set FTOT="{}" + set XMLSCENARIO="{}" + """.format(python, ftot, scenario_xml) + wf.writelines(header_info.replace(" ", "")) # remove the indentation white space + + ftot_script_chunk_a = """ + + REM ============================================== + REM ======== RUN THE FTOT SCRIPT ================= + REM ============================================== + + REM SETUP: SETUP FTOT FOR THE SCENARIO + %PYTHON% %FTOT% %XMLSCENARIO% s || exit /b + + REM FACILITIES: ADD FACILITIES LOCATIONS AND + REM COMMODITY DATA FILES TO THE SCENARIO + %PYTHON% %FTOT% %XMLSCENARIO% f || exit /b + + REM CONNECTIVITY: CONNECT THE FACILITIES TO THE NETWORK AND + REM EXPORT THE FROM GIS TO NETWORKX MODULE + %PYTHON% %FTOT% %XMLSCENARIO% c || exit /b + + REM GRAPH: CREATE THE NETWORKX GRAPHS FROM THE + REM NETWORK AND FACILTIIES + %PYTHON% %FTOT% %XMLSCENARIO% g || exit /b + """ + + wf.writelines(ftot_script_chunk_a.replace(" ", "")) # remove the indentation white space + + if candidate_bool == True: + ftot_script_chunk_b = """ + REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION SETUP DATABASE + %PYTHON% %FTOT% %XMLSCENARIO% oc1 || exit /b + + REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION DEFINE & SOLVE PROBLEM + %PYTHON% %FTOT% %XMLSCENARIO% oc2 || exit /b + + REM OPTMIZATION: PRE-CANDIDATE GENERATION OPTIMIZATION POSTPROCESS + %PYTHON% %FTOT% %XMLSCENARIO% oc3 || exit /b + + REM FACILITIES 2 : ADD FACILITIES LOCATIONS AND + REM COMMODITY DATA FILES TO THE SCENARIO + %PYTHON% %FTOT% %XMLSCENARIO% f2 || exit /b + + REM CONNECTIVITY 2: CONNECT THE FACILITIES TO THE NETWORK AND + REM EXPORT THE FROM GIS TO NETWORKX MODULE + %PYTHON% %FTOT% %XMLSCENARIO% c2 || exit /b + + REM GRAPH 2 : CREATE THE NETWORKX GRAPHS FROM THE + REM NETWORK AND FACILTIIES + %PYTHON% %FTOT% %XMLSCENARIO% g2 || exit /b + """ + wf.writelines(ftot_script_chunk_b.replace(" ", "")) # remove the indentation white space + + + ftot_script_chunk_c = """ + REM STEP OPTIMIZATION: SET UP THE OPTIMIZATION PROBLEM + %PYTHON% %FTOT% %XMLSCENARIO% o1 || exit /b + + REM STEP OPTIMIZATION: BUILD THE OPTIMIZATION PROBLEM AND SOLVE + %PYTHON% %FTOT% %XMLSCENARIO% o2 || exit /b + + REM POST-PROCESSING: POST PROCESS OPTIMAL RESULTS AND PREPARE REPORTING + %PYTHON% %FTOT% %XMLSCENARIO% p || exit /b + + REM REPORT: CREATE REPORTS OF THE CONFIGURATION, RESULTS, AND + REM WARNINGS FROM THE RUN + %PYTHON% %FTOT% %XMLSCENARIO% d || exit /b + + REM MAPS: GENERATE MAPS OF THE SCENARIO FOR EACH STEP + %PYTHON% %FTOT% %XMLSCENARIO% m || exit /b + + REM OPTIONAL MAPPING STEP FOR TIME AND COMMODITY DISTANCES + REM %PYTHON% %FTOT% %XMLSCENARIO% m2 || exit /b + """ + wf.writelines(ftot_script_chunk_c.replace(" ", "")) # remove the indentation white space + + print("file location: {}".format(run_bat_file)) print("all done.") \ No newline at end of file diff --git a/program/tools/scenario_compare_tool.py b/program/tools/scenario_compare_tool.py index 27288ac..148ac0f 100644 --- a/program/tools/scenario_compare_tool.py +++ b/program/tools/scenario_compare_tool.py @@ -1,300 +1,300 @@ -# ------------------------------------------------------------------------------- -# Name: FTOT Scenario Compare Tool -# Purpose: Concatenates Tableau reports and gdb contained in multiple directories. - -# it is a command-line-interface (CLI) tool that prompts the user for the following information: -# - Location of Scenarios to compare -# - Location of directory in which to store scenario comparison output -# ------------------------------------------------------------------------------- - -import arcpy -import os -import csv -import sys -import zipfile -import datetime -import ntpath -import glob -from shutil import copy, rmtree -from six.moves import input - -# ============================================================================== - - -def run_scenario_compare_prep_tool(): - print("FTOT scenario comparison tool") - print("-------------------------------") - print("") - print("") - - scenario_compare_prep() - - -# ============================================================================== - - -def scenario_compare_prep(): - - # get output directory from user: - output_dir = get_output_dir() - # open for user - os.startfile(output_dir) - - # create output gdb - print("start: create tableau gdb") - output_gdb_name = "tableau_output" - os.path.join(output_dir, "tableau_output") - arcpy.CreateFileGDB_management(out_folder_path=output_dir, - out_name=output_gdb_name, out_version="CURRENT") - output_gdb = os.path.join(output_dir, output_gdb_name + ".gdb") - - # create output csv - print("start: Tableau results report") - report_file_name = 'tableau_report.csv' - report_file = os.path.join(output_dir, report_file_name) - # write the first header line - wf = open(report_file, 'w') - header_line = 'scenario_name, table_name, commodity, facility_name, measure, mode, value, units, notes\n' - wf.write(header_line) - - # get user directory to search - # returns list of dirs - scenario_dirs = get_input_dirs() - - # for each dir, get all of the report folders matching the pattern - for a_dir in scenario_dirs: - print("processing: {}".format(a_dir)) - report_dirs = glob.glob(os.path.join(a_dir, "reports", \ - "tableau_report_*_*_*_*-*-*")) - - # skip empty scenarios without report folders - if len(report_dirs) == 0: - print("skipping: {}".format(a_dir)) - continue - - report_dir_dict = [] - for report in report_dirs: - - # skip the csv files...we just want the report folders - if report.endswith('.csv'): - continue - - # inspect the reports folder - path_to, the_file_name = ntpath.split(report) - - the_date = datetime.datetime.strptime(the_file_name[14:], "_%Y_%m_%d_%H-%M-%S") - - report_dir_dict.append((the_file_name, the_date)) - - # find the newest reports folder - # sort by datetime so the most recent is first. - report_dir_dict = sorted(report_dir_dict, key=lambda tup: tup[1], reverse=True) - most_recent_report_file = os.path.join(path_to, report_dir_dict[0][0]) - - # - unzip twbx locally to a temp folder - temp_folder = os.path.join(most_recent_report_file, "temp") - with zipfile.ZipFile(os.path.join(most_recent_report_file, 'tableau_dashboard.twbx'), 'r') as zipObj: - # Extract all the contents of zip file in current directory - zipObj.extractall(temp_folder) - - # copy the TWB and triskelion image file from the unzipped packaged workbook to - # the scenario compare output folder - print("copying the template twb and image files") - copy_list = ["volpeTriskelion.gif", "tableau_dashboard.twb", "parameters_icon.png"] - for a_file in copy_list: - source_loc = os.path.join(temp_folder, a_file) - dest_loc = os.path.join(output_dir, a_file) - try: - copy(source_loc, dest_loc) - except: - print("warning: file {} does not exists.".format(a_file)) - - # - concat tableau_report.csv - print("time to look at the csv file and import ") - csv_in = open(os.path.join(temp_folder,"tableau_report.csv")) - for line in csv_in: - if line.startswith(header_line): - continue - wf.write(line) - csv_in.close() - - # - unzip gdb.zip locally - with zipfile.ZipFile(os.path.join(temp_folder, 'tableau_output.gdb.zip'), - 'r') as zipObj: - # Extract all the contents of zip file in current directory - zipObj.extractall(os.path.join(temp_folder,'tableau_output.gdb')) - - # concat the fc into the output gdb - input_gdb = os.path.join(temp_folder,'tableau_output.gdb') - input_fcs = ["facilities_merge", "optimized_route_segments_dissolved", "optimized_route_segments"] - # try to append the facilities_merge fc. - # if it throws ExecuteError, the FC doesn't exist yet, so do a copy instead. - # the copy only need to be done once to create the first fc. - for fc in input_fcs: - print("processing fc: {}".format(fc)) - try: - # print("try processing fc with append: {} for scenario {}".format(fc, a_dir)) - arcpy.Append_management(os.path.join(input_gdb, fc), - os.path.join(output_gdb, fc), "NO_TEST") - except arcpy.ExecuteError: - # print("except processing fc with copy: {} for scenario {} ".format(fc, a_dir)) - arcpy.CopyFeatures_management(os.path.join(input_gdb, fc), - os.path.join(output_gdb, fc)) - - # remove the temp folder - print("cleaning up temp folder") - rmtree(temp_folder) - - # close merged csv - wf.close() - - # package up the concat files into a compare.twbx - # Create the zip file for writing compressed data - - print('creating archive') - zip_gdb_filename = output_gdb + ".zip" - zf = zipfile.ZipFile(zip_gdb_filename, 'w', zipfile.ZIP_DEFLATED) - - zipgdb(output_gdb, zf) - zf.close() - print('all done zipping') - - # now delete the unzipped gdb - arcpy.Delete_management(output_gdb) - print("deleted unzipped gdb") - - twbx_dashboard_filename = os.path.join(output_dir, "tableau_dashboard.twbx") - zipObj = zipfile.ZipFile(twbx_dashboard_filename, 'w', zipfile.ZIP_DEFLATED) - - # Package the workbook - # need to specify the archive name parameter to avoid the whole path to the file being added to the archive - file_list = ["tableau_dashboard.twb", "tableau_output.gdb.zip", "tableau_report.csv", "volpeTriskelion.gif", - "parameters_icon.png"] - - for a_file in file_list: - - # write the temp file to the zip - zipObj.write(os.path.join(output_dir, a_file), a_file) - - # delete the temp file so its nice and clean. - os.remove(os.path.join(output_dir, a_file)) - - # close the Zip File - zipObj.close() - - -# ============================================================================== - - -def get_input_dirs(): - # recursive? or user specified - # returns list of dirs - print("scenario comparison tool | step 2/2:") - print("-------------------------------") - print("Option 1: recursive directory search ") - print("Option 2: user-specified directories ") - user_choice = input('Enter 1 or 2 or quit: >> ') - if int(user_choice) == 1: - return get_immediate_subdirectories() - elif int(user_choice) == 2: - return get_user_specified_directories() - else: - print("WARNING: not a valid choice! please press enter to continue.") - input("press any key to quit") - - -# ============================================================================== - - -def get_immediate_subdirectories(): - top_dir = "" - print("enter top level directory") - top_dir = input('----------------------> ') - return [os.path.join(top_dir, name) for name in os.listdir(top_dir) - if os.path.isdir(os.path.join(top_dir, name))] - - -# ============================================================================== - - -def get_user_specified_directories(): - # Scenario Directories - list_of_dirs = [] - scenario_counter = 1 - get_dirs = True - while get_dirs: - print("Enter Directory Location for scenario {}:".format(scenario_counter)) - print("Type 'done' if you have entered all of your scenarios") - a_scenario = "" - while not os.path.exists(a_scenario): - a_scenario = input('----------------------> ') - print("USER INPUT: the scenario path: {}".format(a_scenario)) - if a_scenario.lower() == 'done': - if scenario_counter < 2: - print("must define at least two scenario directories for comparison.") - else: - get_dirs = False - break - if not os.path.exists(a_scenario): - print("the following path is not valid. Please enter a valid path to a scenario directory") - print("os.path.exists == False for: {}".format(a_scenario)) - else: - list_of_dirs.append(a_scenario) - scenario_counter += 1 - return list_of_dirs - - -# ============================================================================== - - -def get_output_dir(): - # Scenario Comparison Output Directory - print("scenario comparison tool | step 1/2:") - print("-------------------------------") - print("scenario comparison output directory: ") - scenario_output = "" - scenario_output = input('----------------------> ') - print("USER INPUT: the scenario output path: {}".format(scenario_output)) - if not os.path.exists(scenario_output): - os.makedirs(scenario_output) - return scenario_output - - -# ============================================================================== - - -def clean_file_name(value): - deletechars = '\/:*?"<>|' - for c in deletechars: - value = value.replace(c, '') - return value; - -# ============================================================================== - - -# Function for zipping files -def zipgdb(path, zip_file): - isdir = os.path.isdir - - # Check the contents of the workspace, if it the current - # item is a directory, gets its contents and write them to - # the zip file, otherwise write the current file item to the - # zip file - for each in os.listdir(path): - fullname = path + "/" + each - if not isdir(fullname): - # If the workspace is a file geodatabase, avoid writing out lock - # files as they are unnecessary - if not each.endswith('.lock'): - # Write out the file and give it a relative archive path - try: zip_file.write(fullname, each) - except IOError: None # Ignore any errors in writing file - else: - # Branch for sub-directories - for eachfile in os.listdir(fullname): - if not isdir(eachfile): - if not each.endswith('.lock'): - gp.AddMessage("Adding " + eachfile + " ...") - # Write out the file and give it a relative archive path - try: zip_file.write(fullname + "/" + eachfile, os.path.basename(fullname) + "/" + eachfile) - except IOError: None # Ignore any errors in writing file +# ------------------------------------------------------------------------------- +# Name: FTOT Scenario Compare Tool +# Purpose: Concatenates Tableau reports and gdb contained in multiple directories. + +# it is a command-line-interface (CLI) tool that prompts the user for the following information: +# - Location of Scenarios to compare +# - Location of directory in which to store scenario comparison output +# ------------------------------------------------------------------------------- + +import arcpy +import os +import csv +import sys +import zipfile +import datetime +import ntpath +import glob +from shutil import copy, rmtree +from six.moves import input + +# ============================================================================== + + +def run_scenario_compare_prep_tool(): + print("FTOT scenario comparison tool") + print("-------------------------------") + print("") + print("") + + scenario_compare_prep() + + +# ============================================================================== + + +def scenario_compare_prep(): + + # get output directory from user: + output_dir = get_output_dir() + # open for user + os.startfile(output_dir) + + # create output gdb + print("start: create tableau gdb") + output_gdb_name = "tableau_output" + os.path.join(output_dir, "tableau_output") + arcpy.CreateFileGDB_management(out_folder_path=output_dir, + out_name=output_gdb_name, out_version="CURRENT") + output_gdb = os.path.join(output_dir, output_gdb_name + ".gdb") + + # create output csv + print("start: Tableau results report") + report_file_name = 'tableau_report.csv' + report_file = os.path.join(output_dir, report_file_name) + # write the first header line + wf = open(report_file, 'w') + header_line = 'scenario_name, table_name, commodity, facility_name, measure, mode, value, units, notes\n' + wf.write(header_line) + + # get user directory to search + # returns list of dirs + scenario_dirs = get_input_dirs() + + # for each dir, get all of the report folders matching the pattern + for a_dir in scenario_dirs: + print("processing: {}".format(a_dir)) + report_dirs = glob.glob(os.path.join(a_dir, "reports", \ + "reports_*_*_*_*-*-*")) + + # skip empty scenarios without report folders + if len(report_dirs) == 0: + print("skipping: {}".format(a_dir)) + continue + + report_dir_dict = [] + for report in report_dirs: + + # skip the csv files...we just want the report folders + if report.endswith('.csv'): + continue + + # inspect the reports folder + path_to, the_file_name = ntpath.split(report) + + the_date = datetime.datetime.strptime(the_file_name[7:], "_%Y_%m_%d_%H-%M-%S") + + report_dir_dict.append((the_file_name, the_date)) + + # find the newest reports folder + # sort by datetime so the most recent is first. + report_dir_dict = sorted(report_dir_dict, key=lambda tup: tup[1], reverse=True) + most_recent_report_file = os.path.join(path_to, report_dir_dict[0][0]) + + # - unzip twbx locally to a temp folder + temp_folder = os.path.join(most_recent_report_file, "temp") + with zipfile.ZipFile(os.path.join(most_recent_report_file, 'tableau_dashboard.twbx'), 'r') as zipObj: + # Extract all the contents of zip file in current directory + zipObj.extractall(temp_folder) + + # copy the TWB and triskelion image file from the unzipped packaged workbook to + # the scenario compare output folder + print("copying the template twb and image files") + copy_list = ["volpeTriskelion.gif", "tableau_dashboard.twb", "parameters_icon.png"] + for a_file in copy_list: + source_loc = os.path.join(temp_folder, a_file) + dest_loc = os.path.join(output_dir, a_file) + try: + copy(source_loc, dest_loc) + except: + print("warning: file {} does not exists.".format(a_file)) + + # - concat tableau_report.csv + print("time to look at the csv file and import ") + csv_in = open(os.path.join(temp_folder,"tableau_report.csv")) + for line in csv_in: + if line.startswith(header_line): + continue + wf.write(line) + csv_in.close() + + # - unzip gdb.zip locally + with zipfile.ZipFile(os.path.join(temp_folder, 'tableau_output.gdb.zip'), + 'r') as zipObj: + # Extract all the contents of zip file in current directory + zipObj.extractall(os.path.join(temp_folder,'tableau_output.gdb')) + + # concat the fc into the output gdb + input_gdb = os.path.join(temp_folder,'tableau_output.gdb') + input_fcs = ["facilities_merge", "optimized_route_segments_dissolved", "optimized_route_segments"] + # try to append the facilities_merge fc. + # if it throws ExecuteError, the FC doesn't exist yet, so do a copy instead. + # the copy only need to be done once to create the first fc. + for fc in input_fcs: + print("processing fc: {}".format(fc)) + try: + # print("try processing fc with append: {} for scenario {}".format(fc, a_dir)) + arcpy.Append_management(os.path.join(input_gdb, fc), + os.path.join(output_gdb, fc), "NO_TEST") + except arcpy.ExecuteError: + # print("except processing fc with copy: {} for scenario {} ".format(fc, a_dir)) + arcpy.CopyFeatures_management(os.path.join(input_gdb, fc), + os.path.join(output_gdb, fc)) + + # remove the temp folder + print("cleaning up temp folder") + rmtree(temp_folder) + + # close merged csv + wf.close() + + # package up the concat files into a compare.twbx + # Create the zip file for writing compressed data + + print('creating archive') + zip_gdb_filename = output_gdb + ".zip" + zf = zipfile.ZipFile(zip_gdb_filename, 'w', zipfile.ZIP_DEFLATED) + + zipgdb(output_gdb, zf) + zf.close() + print('all done zipping') + + # now delete the unzipped gdb + arcpy.Delete_management(output_gdb) + print("deleted unzipped gdb") + + twbx_dashboard_filename = os.path.join(output_dir, "tableau_dashboard.twbx") + zipObj = zipfile.ZipFile(twbx_dashboard_filename, 'w', zipfile.ZIP_DEFLATED) + + # Package the workbook + # need to specify the archive name parameter to avoid the whole path to the file being added to the archive + file_list = ["tableau_dashboard.twb", "tableau_output.gdb.zip", "tableau_report.csv", "volpeTriskelion.gif", + "parameters_icon.png"] + + for a_file in file_list: + + # write the temp file to the zip + zipObj.write(os.path.join(output_dir, a_file), a_file) + + # delete the temp file so its nice and clean. + os.remove(os.path.join(output_dir, a_file)) + + # close the Zip File + zipObj.close() + + +# ============================================================================== + + +def get_input_dirs(): + # recursive? or user specified + # returns list of dirs + print("scenario comparison tool | step 2/2:") + print("-------------------------------") + print("Option 1: recursive directory search ") + print("Option 2: user-specified directories ") + user_choice = input('Enter 1 or 2 or quit: >> ') + if int(user_choice) == 1: + return get_immediate_subdirectories() + elif int(user_choice) == 2: + return get_user_specified_directories() + else: + print("WARNING: not a valid choice! please press enter to continue.") + input("press any key to quit") + + +# ============================================================================== + + +def get_immediate_subdirectories(): + top_dir = "" + print("enter top level directory") + top_dir = input('----------------------> ') + return [os.path.join(top_dir, name) for name in os.listdir(top_dir) + if os.path.isdir(os.path.join(top_dir, name))] + + +# ============================================================================== + + +def get_user_specified_directories(): + # Scenario Directories + list_of_dirs = [] + scenario_counter = 1 + get_dirs = True + while get_dirs: + print("Enter Directory Location for scenario {}:".format(scenario_counter)) + print("Type 'done' if you have entered all of your scenarios") + a_scenario = "" + while not os.path.exists(a_scenario): + a_scenario = input('----------------------> ') + print("USER INPUT: the scenario path: {}".format(a_scenario)) + if a_scenario.lower() == 'done': + if scenario_counter < 2: + print("must define at least two scenario directories for comparison.") + else: + get_dirs = False + break + if not os.path.exists(a_scenario): + print("the following path is not valid. Please enter a valid path to a scenario directory") + print("os.path.exists == False for: {}".format(a_scenario)) + else: + list_of_dirs.append(a_scenario) + scenario_counter += 1 + return list_of_dirs + + +# ============================================================================== + + +def get_output_dir(): + # Scenario Comparison Output Directory + print("scenario comparison tool | step 1/2:") + print("-------------------------------") + print("scenario comparison output directory: ") + scenario_output = "" + scenario_output = input('----------------------> ') + print("USER INPUT: the scenario output path: {}".format(scenario_output)) + if not os.path.exists(scenario_output): + os.makedirs(scenario_output) + return scenario_output + + +# ============================================================================== + + +def clean_file_name(value): + deletechars = '\/:*?"<>|' + for c in deletechars: + value = value.replace(c, '') + return value; + +# ============================================================================== + + +# Function for zipping files +def zipgdb(path, zip_file): + isdir = os.path.isdir + + # Check the contents of the workspace, if it the current + # item is a directory, gets its contents and write them to + # the zip file, otherwise write the current file item to the + # zip file + for each in os.listdir(path): + fullname = path + "/" + each + if not isdir(fullname): + # If the workspace is a file geodatabase, avoid writing out lock + # files as they are unnecessary + if not each.endswith('.lock'): + # Write out the file and give it a relative archive path + try: zip_file.write(fullname, each) + except IOError: None # Ignore any errors in writing file + else: + # Branch for sub-directories + for eachfile in os.listdir(fullname): + if not isdir(eachfile): + if not each.endswith('.lock'): + gp.AddMessage("Adding " + eachfile + " ...") + # Write out the file and give it a relative archive path + try: zip_file.write(fullname + "/" + eachfile, os.path.basename(fullname) + "/" + eachfile) + except IOError: None # Ignore any errors in writing file diff --git a/program/tools/xml_text_replacement_tool.py b/program/tools/xml_text_replacement_tool.py index 8fd02a2..323d020 100644 --- a/program/tools/xml_text_replacement_tool.py +++ b/program/tools/xml_text_replacement_tool.py @@ -1,234 +1,234 @@ -# ------------------------------------------------------------------------------- -# Name: XML Text Replacement Tool -# Purpose: Searches for all xml files in a directory and -# replaces the text for a specified element -# ------------------------------------------------------------------------------- - -import os, sys -from six.moves import input -try: - from lxml import etree, objectify -except ImportError: - print ("This script requires the lxml Python library to validate the XML scenario file.") - print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") - print("Exiting...") - sys.exit() - - -# ============================================================================== - -def run(): - print("FTOT XML text replacement tool") - print("-------------------------------") - print("") - print("") - xml_text_replacement() - - -# ============================================================================== - -def xml_text_replacement(): - - print("start: replace text in XMLs") - print("get user inputs") - - # directory containing xmls where a certain element needs to be updated - xml_directory_path = get_directory_path() - - while True: - - # xml element to update - element_to_update = get_element_to_update() - # new text to replace element's existing text - new_text = get_new_text() - - element_path = "" - path_chosen = False - for subdir, dirs, files in os.walk(xml_directory_path): - for file in files: - ext = os.path.splitext(file)[-1].lower() - if ext == ".xml": - full_path_to_xml = os.path.join(subdir, file) - xml_etree = load_scenario_config_file(full_path_to_xml) - print(file) - # identify specific element path - # only runs until a valid path is found for one file - if not path_chosen: - matches = get_all_matches(xml_etree, element_to_update) - if len(matches) > 1: - print("multiple element matches found. please select the desired element path:") - element_path = select_from_menu(matches) - path_chosen = True - elif len(matches) == 1: - element_path = matches[0] - path_chosen = True - do_the_update(xml_etree, element_path, new_text, full_path_to_xml) - - choice = "" - while choice not in ["y","n"]: - print('replace another xml element in this directory? y or n') - choice = input(">> ") - if choice.lower() == 'y': - continue - elif choice.lower() == 'n': - return - - -# ============================================================================== - -def select_from_menu(matches): - # display a menu with cleaned element paths - # and record path selected by user - #------------------------------------------- - - for path in matches: - clean_path = clean_element_path(path) - print("[" + str(matches.index(path)) + "] " + clean_path) - choice = input(">> ") - try: - if int(choice) < 0: raise ValueError - return matches[int(choice)] - except (ValueError, IndexError): - print("not a valid option. please enter a number from the menu.") - return select_from_menu(matches) - - -# ============================================================================== - -def load_scenario_config_file(fullPathToXmlConfigFile): - - if not os.path.exists(fullPathToXmlConfigFile): - raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) - - if fullPathToXmlConfigFile.rfind(".xml") < 0: - raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) - - parser = etree.XMLParser(remove_blank_text=True) - return etree.parse(fullPathToXmlConfigFile, parser) - - -# ============================================================================== - -def clean_element_name(element): - # remove namespace from front of tag, e.g. {FTOT}Scenario --> Scenario - # takes an Element or a String as input - #--------------------------------------------------------------------- - - if type(element) is str: - ns_i = element.find("}") - return element[ns_i + 1:] - else: - ns_i = element.tag.find("}") - return element.tag[ns_i + 1:] - - -# ============================================================================== - -def clean_element_path(path): - # remove namespace from front of elements in path (formatted as string) - # ----------------------------------------------------------------------- - - tagged_elements = path.split("/") - clean_elements = [clean_element_name(elem) for elem in tagged_elements] - return "./" + "/".join(clean_elements) - - -# ============================================================================== - -def do_the_update(xml_etree, element_path, new_text, full_path_to_xml): - # update text of element at specified path - # ---------------------------------------- - - try: - target_elem = xml_etree.findall(element_path)[0] - except (SyntaxError, IndexError): - print("...warning: element not found. no changes made.") - return - - clean_path = clean_element_path(element_path) - print("element path = {}".format(clean_path)) - if len(target_elem) != 0: - raise ValueError("Update stopped. This XML element contains subelements. Text cannot be replaced.") - else: - print("updating the text:") - print ("old text = {}".format(target_elem.text)) - target_elem.text = new_text - print ("new text = {}".format(target_elem.text)) - - save_the_xml_file(full_path_to_xml, xml_etree) - - -# ============================================================================== - -def save_the_xml_file(full_path_to_xml, the_temp_etree): - - with open(full_path_to_xml, 'wb') as wf: - print("writing the file: {} ".format(full_path_to_xml)) - the_temp_etree.write(wf, pretty_print=True) - print("done writing xml file: {}".format(full_path_to_xml)) - - -# ============================================================================== - -def get_all_matches(xml_etree, element_to_update): - # load xml and iterate through all elements to identify possible matches - # return list of candidate paths to elements - #----------------------------------------------------------------------- - - match_paths = [] - item_counter = 0 - for temp_elem in xml_etree.getiterator(): - item_counter += 1 - - # check if the temp_element has the .find attribute - # if it does, then search for the index - # to '{' char at the end of namespace (e.g. {FTOT}Scenario) - # if there is no attribute, we continue b/c its probably a comment. - if not hasattr(temp_elem.tag, 'find'): - continue - clean_temp_elem = clean_element_name(temp_elem) - - # ignore the element if its text is just white space - if temp_elem.text == "": - continue - - # store element path if element is a match - if clean_temp_elem.lower() == element_to_update.lower(): - new_path = xml_etree.getelementpath(temp_elem) - match_paths.append(new_path) - - return match_paths - - -# ============================================================================== - -def get_directory_path(): - - print("directory containing XMLs where a certain element needs to be updated: (drag and drop is fine here)") - xml_dir = "" - while not os.path.exists(xml_dir): - xml_dir = input('----------------------> ') - print("USER INPUT ----------------->: {}".format(xml_dir)) - if not os.path.exists(xml_dir): - print("Path is not valid. Please enter a valid directory path.") - return xml_dir - - -# ============================================================================== - -def get_element_to_update(): - - print('specify XML element to update:') - element = input('----------------------> ') - print("USER INPUT ----------------->: {}".format(element)) - return element - - -# ============================================================================== - -def get_new_text(): - - print('specify new text to replace element\'s existing text:') - text = input('----------------------> ') - print("USER INPUT ----------------->: {}".format(text)) +# ------------------------------------------------------------------------------- +# Name: XML Text Replacement Tool +# Purpose: Searches for all xml files in a directory and +# replaces the text for a specified element +# ------------------------------------------------------------------------------- + +import os, sys +from six.moves import input +try: + from lxml import etree, objectify +except ImportError: + print ("This script requires the lxml Python library to validate the XML scenario file.") + print("Download the library here: https://pypi.python.org/pypi/lxml/2.3") + print("Exiting...") + sys.exit() + + +# ============================================================================== + +def run(): + print("FTOT XML text replacement tool") + print("-------------------------------") + print("") + print("") + xml_text_replacement() + + +# ============================================================================== + +def xml_text_replacement(): + + print("start: replace text in XMLs") + print("get user inputs") + + # directory containing xmls where a certain element needs to be updated + xml_directory_path = get_directory_path() + + while True: + + # xml element to update + element_to_update = get_element_to_update() + # new text to replace element's existing text + new_text = get_new_text() + + element_path = "" + path_chosen = False + for subdir, dirs, files in os.walk(xml_directory_path): + for file in files: + ext = os.path.splitext(file)[-1].lower() + if ext == ".xml": + full_path_to_xml = os.path.join(subdir, file) + xml_etree = load_scenario_config_file(full_path_to_xml) + print(file) + # identify specific element path + # only runs until a valid path is found for one file + if not path_chosen: + matches = get_all_matches(xml_etree, element_to_update) + if len(matches) > 1: + print("multiple element matches found. please select the desired element path:") + element_path = select_from_menu(matches) + path_chosen = True + elif len(matches) == 1: + element_path = matches[0] + path_chosen = True + do_the_update(xml_etree, element_path, new_text, full_path_to_xml) + + choice = "" + while choice not in ["y","n"]: + print('replace another xml element in this directory? y or n') + choice = input(">> ") + if choice.lower() == 'y': + continue + elif choice.lower() == 'n': + return + + +# ============================================================================== + +def select_from_menu(matches): + # display a menu with cleaned element paths + # and record path selected by user + #------------------------------------------- + + for path in matches: + clean_path = clean_element_path(path) + print("[" + str(matches.index(path)) + "] " + clean_path) + choice = input(">> ") + try: + if int(choice) < 0: raise ValueError + return matches[int(choice)] + except (ValueError, IndexError): + print("not a valid option. please enter a number from the menu.") + return select_from_menu(matches) + + +# ============================================================================== + +def load_scenario_config_file(fullPathToXmlConfigFile): + + if not os.path.exists(fullPathToXmlConfigFile): + raise IOError("XML Scenario File {} not found at specified location.".format(fullPathToXmlConfigFile)) + + if fullPathToXmlConfigFile.rfind(".xml") < 0: + raise IOError("XML Scenario File {} is not an XML file type.".format(fullPathToXmlConfigFile)) + + parser = etree.XMLParser(remove_blank_text=True) + return etree.parse(fullPathToXmlConfigFile, parser) + + +# ============================================================================== + +def clean_element_name(element): + # remove namespace from front of tag, e.g. {FTOT}Scenario --> Scenario + # takes an Element or a String as input + #--------------------------------------------------------------------- + + if type(element) is str: + ns_i = element.find("}") + return element[ns_i + 1:] + else: + ns_i = element.tag.find("}") + return element.tag[ns_i + 1:] + + +# ============================================================================== + +def clean_element_path(path): + # remove namespace from front of elements in path (formatted as string) + # ----------------------------------------------------------------------- + + tagged_elements = path.split("/") + clean_elements = [clean_element_name(elem) for elem in tagged_elements] + return "./" + "/".join(clean_elements) + + +# ============================================================================== + +def do_the_update(xml_etree, element_path, new_text, full_path_to_xml): + # update text of element at specified path + # ---------------------------------------- + + try: + target_elem = xml_etree.findall(element_path)[0] + except (SyntaxError, IndexError): + print("...warning: element not found. no changes made.") + return + + clean_path = clean_element_path(element_path) + print("element path = {}".format(clean_path)) + if len(target_elem) != 0: + raise ValueError("Update stopped. This XML element contains subelements. Text cannot be replaced.") + else: + print("updating the text:") + print ("old text = {}".format(target_elem.text)) + target_elem.text = new_text + print ("new text = {}".format(target_elem.text)) + + save_the_xml_file(full_path_to_xml, xml_etree) + + +# ============================================================================== + +def save_the_xml_file(full_path_to_xml, the_temp_etree): + + with open(full_path_to_xml, 'wb') as wf: + print("writing the file: {} ".format(full_path_to_xml)) + the_temp_etree.write(wf, pretty_print=True) + print("done writing xml file: {}".format(full_path_to_xml)) + + +# ============================================================================== + +def get_all_matches(xml_etree, element_to_update): + # load xml and iterate through all elements to identify possible matches + # return list of candidate paths to elements + #----------------------------------------------------------------------- + + match_paths = [] + item_counter = 0 + for temp_elem in xml_etree.getiterator(): + item_counter += 1 + + # check if the temp_element has the .find attribute + # if it does, then search for the index + # to '{' char at the end of namespace (e.g. {FTOT}Scenario) + # if there is no attribute, we continue b/c its probably a comment. + if not hasattr(temp_elem.tag, 'find'): + continue + clean_temp_elem = clean_element_name(temp_elem) + + # ignore the element if its text is just white space + if temp_elem.text == "": + continue + + # store element path if element is a match + if clean_temp_elem.lower() == element_to_update.lower(): + new_path = xml_etree.getelementpath(temp_elem) + match_paths.append(new_path) + + return match_paths + + +# ============================================================================== + +def get_directory_path(): + + print("directory containing XMLs where a certain element needs to be updated: (drag and drop is fine here)") + xml_dir = "" + while not os.path.exists(xml_dir): + xml_dir = input('----------------------> ') + print("USER INPUT ----------------->: {}".format(xml_dir)) + if not os.path.exists(xml_dir): + print("Path is not valid. Please enter a valid directory path.") + return xml_dir + + +# ============================================================================== + +def get_element_to_update(): + + print('specify XML element to update:') + element = input('----------------------> ') + print("USER INPUT ----------------->: {}".format(element)) + return element + + +# ============================================================================== + +def get_new_text(): + + print('specify new text to replace element\'s existing text:') + text = input('----------------------> ') + print("USER INPUT ----------------->: {}".format(text)) return text \ No newline at end of file diff --git a/simple_setup.bat b/simple_setup.bat index 6951521..da90d68 100644 --- a/simple_setup.bat +++ b/simple_setup.bat @@ -1,35 +1,35 @@ -@ECHO OFF -set PYTHONDONTWRITEBYTECODE=1 - -set CONDA="%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe" -IF NOT EXIST %CONDA% ( - SET CONDA="%LOCALAPPDATA%\Programs\ArcGIS\Pro\bin\Python\Scripts\conda.exe" -) -set NEWENV="C:\FTOT\python3_env" -set NEWPYTHON="C:\FTOT\python3_env\python.exe" - -echo Starting FTOT installation - -echo Checking if directory %NEWENV% already exists -IF EXIST %NEWENV% ( - echo Warning: directory %NEWENV% already exists. If you have previously installed FTOT, this is expected. - echo Continuing will delete the existing FTOT Python environment and ensure that the new environment - echo is based on the latest FTOT requirements and your current version of ArcGIS Pro. - echo If you do not want to proceed, close the window to exit. - pause - rmdir /q /s %NEWENV% - echo Deleting existing directory -) - -echo Cloning ArcGIS Pro Python environment. This may take a few minutes... -%CONDA% create --clone arcgispro-py3 --prefix %NEWENV% -echo New Python executable at: %NEWPYTHON% - -echo Installing dependencies -%NEWPYTHON% -m pip install --no-warn-script-location pint -%NEWPYTHON% -m pip install --no-warn-script-location pulp -%NEWPYTHON% -m pip install --no-warn-script-location lxml -%NEWPYTHON% -m pip install --no-warn-script-location imageio - -echo Complete. -pause +@ECHO OFF +set PYTHONDONTWRITEBYTECODE=1 + +set CONDA="%PROGRAMFILES%\ArcGIS\Pro\bin\Python\Scripts\conda.exe" +IF NOT EXIST %CONDA% ( + SET CONDA="%LOCALAPPDATA%\Programs\ArcGIS\Pro\bin\Python\Scripts\conda.exe" +) +set NEWENV="C:\FTOT\python3_env" +set NEWPYTHON="C:\FTOT\python3_env\python.exe" + +echo Starting FTOT installation + +echo Checking if directory %NEWENV% already exists +IF EXIST %NEWENV% ( + echo Warning: directory %NEWENV% already exists. If you have previously installed FTOT, this is expected. + echo Continuing will delete the existing FTOT Python environment and ensure that the new environment + echo is based on the latest FTOT requirements and your current version of ArcGIS Pro. + echo If you do not want to proceed, close the window to exit. + pause + rmdir /q /s %NEWENV% + echo Deleting existing directory +) + +echo Cloning ArcGIS Pro Python environment. This may take a few minutes... +%CONDA% create --clone arcgispro-py3 --prefix %NEWENV% +echo New Python executable at: %NEWPYTHON% + +echo Installing dependencies +%NEWPYTHON% -m pip install --no-warn-script-location pint +%NEWPYTHON% -m pip install --no-warn-script-location pulp +%NEWPYTHON% -m pip install --no-warn-script-location lxml +%NEWPYTHON% -m pip install --no-warn-script-location imageio + +echo Complete. +pause