Merge pull request #7 from danyoungday/deployment

Setting up app for deployment

.dockerignore

@@ -0,0 +1,10 @@
+en-roads-sdk-v24.6.0-beta1
+*.zip
+__pycache__
+.DS_Store
+*.pt
+temp/
+.vscode
+results/
+
+*.ipynb

.github/workflows/enroads.yml

@@ -0,0 +1,37 @@
+# This runs the unit tests for the En-ROADS use case
+
+name: enroads Use Case
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint flake8
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Download En-ROADS sdk
+      env:
+        ENROADS_ID: ${{ secrets.ENROADS_ID }}
+        ENROADS_PASSWORD: ${{ secrets.ENROADS_PASSWORD }}
+        ENROADS_URL: ${{ secrets.ENROADS_URL }}
+      run: python -m enroadspy.download_sdk
+    - name: Lint with PyLint
+      run: pylint .
+    - name: Lint with Flake8
+      run: flake8
+    - name: Run unit tests
+      run: python -m unittest
+

.gitignore

@@ -5,4 +5,6 @@ __pycache__
 *.pt
 temp/
 .vscode
-results/
+results/
+
+!app/results

.pylintrc

@@ -1,7 +1,13 @@
 [MASTER]
 
+ignore=inputSpecs.py
+
+recursive=y
+
 max-line-length=120
 
 suggestion-mode=yes
 
-good-names=X,F
+good-names=X,F,X0
+
+fail-under=9.8

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+
+ARG ENROADS_URL
+ARG ENROADS_ID
+ARG ENROADS_PASSWORD
+
+WORKDIR /en-roads-py
+
+# Debian basics and cleaning up in one RUN statement to reduce image size
+RUN apt-get update -y && \
+    apt-get install --no-install-recommends curl git gcc g++ make clang -y && \
+    rm -rf /var/lib/apt/lists/* 
+
+# Dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy source files over
+COPY . .
+
+# Download En-ROADS SDK and extract it
+ENV ENROADS_URL=$ENROADS_URL
+ENV ENROADS_ID=$ENROADS_ID
+ENV ENROADS_PASSWORD=$ENROADS_PASSWORD
+RUN python -m enroadspy.download_sdk
+
+# Expose Flask (Dash) port
+EXPOSE 4057
+
+# Run main UI
+ENTRYPOINT ["gunicorn", "-b", "0.0.0.0:4057", "--timeout", "45", "app.app:server"]

README.md

@@ -4,7 +4,7 @@ Immediate action is required to combat climate change. The technology behind Cog
 
 ## En-ROADS Wrapper
 
-En-ROADS is a climate change simulator developed by Climate Interactive. We have created a wrapper around the SDK to make it simple to use in a Python application. See `enroads_runner.py` for the main class that runs the SDK. The SDK is not included in this repository and must be requested from Climate Interactive.
+En-ROADS is a climate change simulator developed by Climate Interactive. We have created a wrapper around the SDK to make it simple to use in a Python application which can be found in `enroadspy`. See `enroads_runner.py` for the main class that runs the SDK. The SDK is not included in this repository and must be requested from Climate Interactive.
 
 The input data format is a crazy long JSON object which I copied out of the source code, pasted into `inputSpecs.py`, and parsed into `inputSpecs.jsonl`. This format is used by the rest of the code.
 

app/app.py

@@ -28,6 +28,7 @@
 
 # Initialize the Dash app
 app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.BOOTSTRAP, "assets/styles.css"])
+server = app.server
 app.title = "Climate Change Decision Making"
 
 context_component.register_callbacks(app)
@@ -49,4 +50,4 @@
 
 # Run the app
 if __name__ == '__main__':
-    app.run_server(debug=True)
+    app.run_server(host='0.0.0.0', debug=False, port=4057, use_reloader=False, threaded=True)

app/components/context.py

@@ -8,6 +8,8 @@
 import plotly.express as px
 import plotly.graph_objects as go
 
+from enroadspy import load_input_specs
+
 
 class ContextComponent():
     """
@@ -27,7 +29,7 @@ def __init__(self):
 
         # Round context df here instead of automatically by Dash so that we know for sure how it's rounding.
         self.context_df = pd.read_csv("experiments/scenarios/gdp_context.csv")
-        input_specs = pd.read_json("inputSpecs.jsonl", lines=True, precise_float=True)
+        input_specs = load_input_specs()
         for col in self.context_cols:
             row = input_specs[input_specs["varId"] == col].iloc[0]
             step = row["step"]
@@ -88,7 +90,7 @@ def create_context_div(self):
         """
         Creates div showing context scatter plot next to context sliders.
         """
-        input_specs = pd.read_json("inputSpecs.jsonl", lines=True, precise_float=True)
+        input_specs = load_input_specs()
         sliders = []
         for i, (context_col, varname) in enumerate(zip(self.context_cols, self.varnames)):
             row = input_specs[input_specs["varId"] == context_col].iloc[0]

app/components/link.py

@@ -6,7 +6,7 @@
 import pandas as pd
 import plotly.graph_objects as go
 
-from generate_url import actions_to_url
+from enroadspy.generate_url import actions_to_url
 
 
 class LinkComponent():

app/components/outcome.py

@@ -1,6 +1,8 @@
 """
 OutcomeComponent class for the outcome section of the app.
 """
+import sys
+
 from dash import Input, Output, html, dcc
 import dash_bootstrap_components as dbc
 import pandas as pd

app/results/config.json

@@ -0,0 +1 @@
+{"n_generations": 100, "pop_size": 100, "crowding_func": "mnn", "actions": ["_source_subsidy_delivered_coal_tce", "_source_subsidy_start_time_delivered_coal", "_source_subsidy_stop_time_delivered_coal", "_no_new_coal", "_year_of_no_new_capacity_coal", "_utilization_adjustment_factor_delivered_coal", "_utilization_policy_start_time_delivered_coal", "_utilization_policy_stop_time_delivered_coal", "_target_accelerated_retirement_rate_electric_coal", "_source_subsidy_delivered_oil_boe", "_source_subsidy_start_time_delivered_oil", "_source_subsidy_stop_time_delivered_oil", "_no_new_oil", "_year_of_no_new_capacity_oil", "_utilization_adjustment_factor_delivered_oil", "_utilization_policy_start_time_delivered_oil", "_utilization_policy_stop_time_delivered_oil", "_source_subsidy_delivered_gas_mcf", "_source_subsidy_start_time_delivered_gas", "_source_subsidy_stop_time_delivered_gas", "_no_new_gas", "_year_of_no_new_capacity_gas", "_utilization_adjustment_factor_delivered_gas", "_utilization_policy_start_time_delivered_gas", "_utilization_policy_stop_time_delivered_gas", "_source_subsidy_renewables_kwh", "_source_subsidy_start_time_renewables", "_source_subsidy_stop_time_renewables", "_use_subsidies_by_feedstock", "_source_subsidy_delivered_bio_boe", "_source_subsidy_start_time_delivered_bio", "_source_subsidy_stop_time_delivered_bio", "_no_new_bio", "_year_of_no_new_capacity_bio", "_wood_feedstock_subsidy_boe", "_crop_feedstock_subsidy_boe", "_other_feedstock_subsidy_boe", "_source_subsidy_nuclear_kwh", "_source_subsidy_start_time_nuclear", "_source_subsidy_stop_time_nuclear", "_carbon_tax_initial_target", "_carbon_tax_phase_1_start", "_carbon_tax_time_to_achieve_initial_target", "_carbon_tax_final_target", "_carbon_tax_phase_3_start", "_carbon_tax_time_to_achieve_final_target", "_apply_carbon_tax_to_biofuels", "_ccs_carbon_tax_qualifier", "_qualifying_path_renewables", "_qualifying_path_nuclear", "_qualifying_path_new_zero_carbon", "_qualifying_path_beccs", "_qualifying_path_bioenergy", "_qualifying_path_fossil_ccs", "_qualifying_path_gas", "_electric_standard_active", "_electric_standard_target", "_electric_standard_start_year", "_electric_standard_target_time", "_emissions_performance_standard", "_performance_standard_time"], "outcomes": {"Temperature above 1.5C": true, "Max cost of energy": true, "Government net revenue below zero": false, "Total energy below baseline": false}, "save_path": "results/pymoo/context-updated"}

app/utils.py

@@ -3,12 +3,12 @@
 """
 import json
 
-import dill
+import numpy as np
 import pandas as pd
 from sklearn.preprocessing import StandardScaler
 import torch
 
-from enroads_runner import EnroadsRunner
+from enroadspy.enroads_runner import EnroadsRunner
 from evolution.candidate import Candidate
 from evolution.outcomes.outcome_manager import OutcomeManager
 
@@ -18,20 +18,18 @@ class EvolutionHandler():
     Handles evolution results and running of prescriptors for the app.
     """
     def __init__(self):
-        save_path = "results/pymoo/context-updated"
+        save_path = "app/results"
         with open(save_path + "/config.json", 'r', encoding="utf-8") as f:
             config = json.load(f)
 
         self.actions = config["actions"]
         self.outcomes = config["outcomes"]
         # TODO: Make this not hard-coded
         self.model_params = {"in_size": 4, "hidden_size": 16, "out_size": len(self.actions)}
+        self.device = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
 
-        with open(save_path + "/results", 'rb') as f:
-            res = dill.load(f)
-
-        self.X = res.X
-        self.F = res.F
+        self.X = np.load(save_path + "/X.npy")
+        self.F = np.load(save_path + "/F.npy")
 
         context_df = pd.read_csv("experiments/scenarios/gdp_context.csv")
         self.context_df = context_df.drop(columns=["F", "scenario"])
@@ -61,7 +59,7 @@ def load_initial_metrics_df(self):
             baseline_metrics = self.outcome_manager.process_outcomes(context_dict, baseline_outcomes)
             for outcome, val in baseline_metrics.items():
                 baseline_metrics_avg[outcome] += val
-        
+
         # Finish preprocessing baseline metrics
         for outcome in self.outcomes:
             baseline_metrics_avg[outcome] /= len(self.context_df)
@@ -89,7 +87,7 @@ def prescribe_all(self, context_dict: dict[str, float]):
             # Process context_dict into tensor
             context_list = [context_dict[context] for context in self.context_df.columns]
             context_scaled = self.scaler.transform([context_list])
-            context_tensor = torch.tensor(context_scaled, dtype=torch.float32, device="mps")
+            context_tensor = torch.tensor(context_scaled, dtype=torch.float32, device=self.device)
             actions_dict = candidate.prescribe(context_tensor)[0]
             actions_dict.update(context_dict)
             context_actions_dicts.append(actions_dict)

enroadspy/__init__.py

@@ -0,0 +1,14 @@
+"""
+Universal way to gain access to the input specs for the enroads model.
+"""
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_input_specs() -> pd.DataFrame:
+    """
+    Loads the input specs for the En-ROADS model from the inputSpecs.jsonl file.
+    We make sure precise_float=True so that we get exact floats like 15 instead of 15.00001.
+    """
+    return pd.read_json(Path("enroadspy/inputSpecs.jsonl"), lines=True, precise_float=True)

enroadspy/download_sdk.py

@@ -0,0 +1,43 @@
+"""
+Setup script to download the En-ROADS SDK. This is used for app deployment and testing.
+"""
+import os
+import zipfile
+
+import requests
+
+
+def main():
+    """
+    Downloads en-roads sdk and extracts it.
+    If the sdk already exists, we do nothing.
+    If we already have the zip file but no SDK, we just extract the zip file.
+    """
+    zip_path = "enroadspy/en-roads-sdk-v24.6.0-beta1.zip"
+    sdk_path = "enroadspy/"
+
+    if os.path.exists(sdk_path + "en-roads-sdk-v24.6.0-beta1"):
+        print("SDK already exists.")
+        return
+
+    if not os.path.exists(zip_path):
+        url = os.getenv("ENROADS_URL")
+        username = os.getenv("ENROADS_ID")
+        password = os.getenv("ENROADS_PASSWORD")
+        assert username is not None and password is not None, \
+            "Please set the ENROADS_ID and ENROADS_PASSWORD environment variables. \
+            To get access to them go to https://en-roads.climateinteractive.org/ and sign up."
+
+        r = requests.get(url, auth=(username, password), timeout=60)
+
+        if r.status_code == 200:
+            with open(zip_path, "wb") as out:
+                for bits in r.iter_content():
+                    out.write(bits)
+
+    with zipfile.ZipFile(zip_path, "r") as zip_ref:
+        zip_ref.extractall(sdk_path)
+
+
+if __name__ == "__main__":
+    main()

enroads_runner.py → enroadspy/enroads_runner.py

@@ -8,21 +8,23 @@
 import numpy as np
 import pandas as pd
 
+from enroadspy import load_input_specs
+
 
 class EnroadsRunner():
     """
     Class that handles the running of the En-ROADS simulator.
     """
     def __init__(self):
-        self.input_specs = pd.read_json("inputSpecs.jsonl", lines=True, precise_float=True)
+        self.input_specs = load_input_specs()
         self.compile_enroads()
 
     def compile_enroads(self):
         """
         Compiles the en-roads model.
         Make sure you extracted the zip file in the current directory.
         """
-        subprocess.run(["make"], cwd="en-roads-sdk-v24.6.0-beta1/c", check=True)
+        subprocess.run(["make"], cwd="./enroadspy/en-roads-sdk-v24.6.0-beta1/c", check=True)
 
     def format_string_input(self, value, decimal):
         """
@@ -65,6 +67,23 @@ def construct_enroads_input(self, inputs: dict[str, float]):
         return input_str
     # pylint: enable=no-member
 
+    def check_input_string(self, input_str: str) -> bool:
+        """
+        Checks if the input string is valid for security purposes.
+        1. Makes sure the input string is below a certain size in bytes (10,000).
+        2. Makes sure the input string's values are numeric.
+        """
+        if len(input_str.encode('utf-8')) > 10000:
+            return False
+        for pair in input_str.split(" "):
+            try:
+                idx, val = pair.split(":")
+                int(idx)
+                float(val)
+            except ValueError:
+                return False
+        return True
+
     def run_enroads(self, input_str=None):
         """
         Simple function to run the enroads simulator. A temporary file is created storing our input string as the
@@ -75,7 +94,10 @@ def run_enroads(self, input_str=None):
             index number from the value number with no spaces. Index numbers are zero-based.
         NOTE: The indices are the line numbers in inputSpecs.jsonl starting from 0, NOT the id column.
         """
-        command = ["./en-roads-sdk-v24.6.0-beta1/c/enroads"]
+        if input_str and not self.check_input_string(input_str):
+            raise ValueError("Invalid input string")
+
+        command = ["./enroadspy/en-roads-sdk-v24.6.0-beta1/c/enroads"]
         if input_str:
             with tempfile.NamedTemporaryFile(mode="w+", delete=True) as temp_file:
                 temp_file.write(input_str)
@@ -87,8 +109,7 @@ def run_enroads(self, input_str=None):
 
         if result.returncode == 0:
             return result.stdout
-        else:
-            raise ValueError(f"Enroads failed with error code {result.returncode} and message {result.stderr}")
+        raise ValueError(f"Enroads failed with error code {result.returncode} and message {result.stderr}")
 
     def evaluate_actions(self, actions_dict: dict[str, str]):
         """