Notebook_11_create_input_files_for_MOO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "467929a2",
   "metadata": {},
   "source": [
    "## Creation of input files for solving a multiobjective dam portfolio selection problem on Burmese hydroelectric dams\n",
    "### T. Janus\n",
    "### 05/01/2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a041fedb",
   "metadata": {},
   "source": [
    "## TODO:\n",
    "Change `Mong Ton` to `MongTon` in the Salween Pywr models \n",
    "\n",
    "## TO FIX:\n",
    "#### 1. The missing nodes problem\n",
    "#### 2. The wrong mapping between ifc reservoirs and the labels assigned for MOO during file creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4000c23b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import annotations\n",
    "from typing import List, Tuple, Set\n",
    "from dataclasses import dataclass, field\n",
    "import pathlib\n",
    "import copy\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import string\n",
    "import pickle\n",
    "import lib.graph as graphtools\n",
    "from lib.graph import DamNetwork, NetworkSimplifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b00e0cc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Pywr model edges (to construct graph representations of pywr models)\n",
    "salween_edges_path = pathlib.Path(\"inputs/pywr_model_structs/salween_edges.json\")\n",
    "sittaung_edges_path = pathlib.Path(\"inputs/pywr_model_structs/sittaung_edges.json\")\n",
    "irrawaddy_edges_path = pathlib.Path(\"inputs/pywr_model_structs/irrawaddy_edges.json\")\n",
    "salween_edges = graphtools.get_model_edges(salween_edges_path)\n",
    "sittaung_edges = graphtools.get_model_edges(sittaung_edges_path)\n",
    "irrawaddy_edges = graphtools.get_model_edges(irrawaddy_edges_path)\n",
    "# Get pywr model node coordinates (for plotting model graphs)\n",
    "salween_coordinates = graphtools.get_model_coordinates(\n",
    "    pathlib.Path(\"inputs/pywr_model_structs/salween_coordinates.json\"))\n",
    "sittaung_coordinates = graphtools.get_model_coordinates(\n",
    "    pathlib.Path(\"inputs/pywr_model_structs/sittaung_coordinates.json\"))\n",
    "irrawaddy_coordinates = graphtools.get_model_coordinates(\n",
    "    pathlib.Path(\"inputs/pywr_model_structs/irrawaddy_coordinates.json\"))\n",
    "# CSV file with IFC and Pywr dam/reservoir names\n",
    "ifc_pywr_map_path = pathlib.Path(\"config/ifc_pywr_name_map.csv\")\n",
    "# Dam/reservoir data\n",
    "ifc_id_map_path = pathlib.Path(\"outputs/moo/all_hp.csv\")\n",
    "# Get mappings between pywr and ifc names and ifc names and ifc ids\n",
    "pywr_ifc_map = graphtools.pywr_ifc_map_from_csv(ifc_pywr_map_path)\n",
    "ifc_name_ifc_id_map = graphtools.ifc_name_to_ifc_id_from_csv(ifc_id_map_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbceb44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class TestMOOInputData:\n",
    "    \"\"\"Class for making sure that graph representation and the names/ids/structure\n",
    "    represent the true data/information before they're used to generate MOO\n",
    "    input data\"\"\"\n",
    "    network: DamNetwork\n",
    "    dam_data_file: pathlib.Path = pathlib.Path(\n",
    "        \"outputs/moo/all_hp.csv\")\n",
    "    dam_data: pd.DataFrame = pd.DataFrame()\n",
    "    \n",
    "    def __post_init__(self) -> None:\n",
    "        \"\"\" \"\"\"\n",
    "        self.dam_data = pd.read_csv(self.dam_data_file)\n",
    "\n",
    "    @property\n",
    "    def true_data_num_duplicates(self) -> int:\n",
    "        \"\"\" \"\"\"\n",
    "        ifc_id_series = self.dam_data['ifc_id']\n",
    "        return len(ifc_id_series) - len(set(ifc_id_series))\n",
    "\n",
    "    @property\n",
    "    def true_data_num_dams(self) -> int:\n",
    "        return len(set(self.dam_data['ifc_id']))\n",
    "\n",
    "    @property\n",
    "    def true_data_ifc_ids(self) -> List[int]:\n",
    "        \"\"\" \"\"\"\n",
    "        return list(set(self.dam_data['ifc_id']))\n",
    "    \n",
    "    @property\n",
    "    def network_nodes(self) -> List:\n",
    "        return self.network.get_nodes()\n",
    "    \n",
    "    def test_dams_in_graph(\n",
    "            self, graph_type: str = \"node-centric\", dam_id_type: str = \"ifc\"):\n",
    "        \"\"\" \"\"\"\n",
    "        if dam_id_type != \"ifc\":\n",
    "            ...\n",
    "\n",
    "        if graph_type == 'node-centric':\n",
    "            graph_nodes = set(self.network.get_nodes())\n",
    "            nodes_true = set(self.true_data_ifc_ids)\n",
    "\n",
    "            if graph_nodes != nodes_true:\n",
    "                print(\"Graph nodes and db nodes not equal\")\n",
    "                print(f\"Nodes missing in graph: {nodes_true - graph_nodes}\")\n",
    "                print(f\"Extra nodes in graph not in db: {graph_nodes - nodes_true}\")\n",
    "            else:\n",
    "                print(\"Graph data checks out\")\n",
    "\n",
    "        elif graph_type == 'edge-centric':\n",
    "            \"\"\" \"\"\"\n",
    "            missing_dam_ids: List[int] = []\n",
    "            for dam_id in self.true_data_ifc_ids:\n",
    "                edges_with_dam = self.network.find_edges_containing_node(\n",
    "                    node_id = dam_id)\n",
    "                if not edges_with_dam:\n",
    "                    missing_dam_ids.append(dam_id)\n",
    "            if missing_dam_ids:\n",
    "                print(\"Graph does not contain all required nodes\")\n",
    "                print(f\"Nodes missing in graph: {missing_dam_ids}\")\n",
    "            else:\n",
    "                print(\"Graph data checks out\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20113d0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class NodeToEdgeConverter:\n",
    "    \"\"\"Class for changing the representation of the graph in which nodes represent dams\n",
    "    and edges represent connections between dams through the network to the representation\n",
    "    in which nodes are river (water) bodies upstream and downstream of the dam and the dam\n",
    "    is represented as an edge between those nodes.\n",
    "    The change of graph representation is required for specifing optimization problem as value\n",
    "    on a tree network.\"\"\"\n",
    "    network: DamNetwork\n",
    "    verbose: bool = False\n",
    "\n",
    "    def __post_init__(self) -> None:\n",
    "        self.network = copy.deepcopy(self.network)\n",
    "        \n",
    "    @property\n",
    "    def edge_data(self, field: str = 'dam_id') -> List[str]:\n",
    "        \"\"\"Get data associated with edges spanning between nodes. In dam network, edges\n",
    "        represent dams installed on river stretches\"\"\"\n",
    "        edge_data = self.network.graph.edges(data=True)\n",
    "        return [edge[2][field] for edge in edge_data]\n",
    "     \n",
    "    def _generate_random_int(\n",
    "            self, minimum_number: int = 400,  maximum_number: int = 800) -> int:\n",
    "        \"\"\" \"\"\"\n",
    "        return random.randint(minimum_number, maximum_number)\n",
    "        \n",
    "    def _add_dummy_edges(\n",
    "            self, minimum_id: int, maximum_id: int, add_upstream: bool = True,\n",
    "            add_downstream: bool = True) -> DamNetwork:\n",
    "        \"\"\"Add upstream nodes and edges to root nodes and downstream nodes and edges to leaf nodes\n",
    "        Used to represent upstream and dowsnstream river stretches of dams before converting the graph\n",
    "        that represents dams as nodes to the graph in which dams are adges and node are river sections\n",
    "        upstream and downstream of the dam\"\"\"\n",
    "        generated_numbers: Set[int] = set()\n",
    "        if add_upstream:\n",
    "            for root_node in self.network.root_nodes:\n",
    "                while (upstream_node_name := self._generate_random_int(minimum_id, maximum_id))\\\n",
    "                        not in generated_numbers:   \n",
    "                    self.network.graph.add_node(upstream_node_name)\n",
    "                    self.network.graph.add_edge(upstream_node_name, root_node)\n",
    "                    generated_numbers.add(upstream_node_name)\n",
    "                    if self.verbose:\n",
    "                        print(\"Added node: {upstream_node_name}, and edge {upstream_node_name} -> {root_node}\")\n",
    "                    break\n",
    "        if add_downstream:\n",
    "            for leaf_node in self.network.leaf_nodes:\n",
    "                while (downstream_node_name := self._generate_random_int(minimum_id, maximum_id))\\\n",
    "                        not in generated_numbers:\n",
    "                    self.network.graph.add_node(downstream_node_name)\n",
    "                    self.network.graph.add_edge(leaf_node, downstream_node_name)\n",
    "                    generated_numbers.add(downstream_node_name)\n",
    "                    if self.verbose:\n",
    "                        print(\"Added node: {downstream_node_name}, and edge {leaf_node} -> {downstream_node_name}\")\n",
    "                    break\n",
    "        return self.network\n",
    "    \n",
    "    def _reverse_graph(self, inplace: bool = True) -> nx.Graph:\n",
    "        \"\"\" \"\"\"\n",
    "        G_rev = self.network.graph.reverse()\n",
    "        if inplace:\n",
    "            self.network.graph = G_rev\n",
    "        return G_rev\n",
    "\n",
    "    def _remove_duplicates(self) -> DamNetwork:\n",
    "        \"\"\" \"\"\"\n",
    "        # Define a dictionary that stores hashable edge data as keys and upstream and downstream \n",
    "        # nodes as values\n",
    "        seen_edges = dict()\n",
    "        old_dup_edges = set()\n",
    "        old_out_edges = set()\n",
    "        old_nodes = set()\n",
    "\n",
    "        for edge in copy.deepcopy(self.network.graph.edges(data=True)):\n",
    "            # Extract edge attributes for comparison. Skip edges with empty data\n",
    "            # Store information about edges in a hash-table.\n",
    "            # If edge already present in the hash-table then the edge is a duplicate and needs to be removed\n",
    "            edge_data = edge[2]\n",
    "            if not edge_data:\n",
    "                continue\n",
    "            \n",
    "            edge_data_hashable = tuple(sorted(edge_data.items()))\n",
    "\n",
    "            if edge_data_hashable not in seen_edges.keys():\n",
    "                seen_edges[edge_data_hashable] = (edge[0], edge[1])\n",
    "            else:\n",
    "                # Get duplicate edge data\n",
    "                dup_edge_source, dup_edge_target = edge[0], edge[1]\n",
    "                seen_edge = seen_edges[edge_data_hashable]\n",
    "                seen_edge_source, seen_edge_target = seen_edge[0], seen_edge[1]\n",
    "                \n",
    "                if dup_edge_source != seen_edge_source:\n",
    "                    print(f\"Problem with source edge {dup_edge_source} - {dup_edge_target} and\\n\")\n",
    "                    print(f\"{seen_edge_source} - {seen_edge_target}\")\n",
    "                    print(\"Skipping\")\n",
    "                    continue\n",
    "                    #raise ValueError(\"Only edges having the same parents node can be duplicated.\")\n",
    "                    \n",
    "                # Find edges going out of the bottom node of the duplicated node\n",
    "                outgoing_edges = list(self.network.graph.out_edges(dup_edge_target, data=True))\n",
    "\n",
    "                for outgoing_edge in outgoing_edges:\n",
    "                    new_outgoing_edge = (seen_edge_target, outgoing_edge[1], outgoing_edge[2])\n",
    "                    # Add the new outgoing edge\n",
    "                    if self.verbose:\n",
    "                        print(f\"Adding new edge {new_outgoing_edge[0]} - {new_outgoing_edge[1]}\")\n",
    "                    self.network.graph.add_edge(\n",
    "                        new_outgoing_edge[0], new_outgoing_edge[1],  **new_outgoing_edge[2])\n",
    "                    # Remove the old outgoing edge\n",
    "                    #if verbose:\n",
    "                    #    print(f\"Removing old edge {outgoing_edge[0]} - {outgoing_edge[1]}\")\n",
    "                    old_out_edges.add((outgoing_edge[0], outgoing_edge[1]))\n",
    "                    #self.network.graph.remove_edge(outgoing_edge[0], outgoing_edge[1])\n",
    "                    \n",
    "                old_dup_edges.add((dup_edge_source, dup_edge_target))\n",
    "                old_nodes.add(dup_edge_target)   \n",
    "                #self.network.graph.remove_edge(dup_edge_source, dup_edge_target)\n",
    "                #self.network.graph.remove_node(dup_edge_target)\n",
    "                \n",
    "        # Remove old edges and old nodes\n",
    "        for edge in old_out_edges:\n",
    "            try:\n",
    "                self.network.graph.remove_edge(edge[0], edge[1])\n",
    "                if self.verbose:\n",
    "                    print(f\"Removing old edge {edge[0]} - {edge[1]}\")\n",
    "            except nx.NetworkXError:\n",
    "                pass\n",
    "        for edge in old_dup_edges:\n",
    "            try:\n",
    "                self.network.graph.remove_edge(edge[0], edge[1])\n",
    "                if self.verbose:\n",
    "                    print(f\"Removing duplicated edge {edge[0]} - {edge[1]}\")\n",
    "            except nx.NetworkXError:\n",
    "                pass\n",
    "        for node in old_nodes:\n",
    "            try:\n",
    "                self.network.graph.remove_node(node)\n",
    "                if self.verbose:\n",
    "                    print(f\"Removing node {node}\")\n",
    "            except nx.NetworkXError:\n",
    "                pass\n",
    "        \n",
    "        # Sanitize the graph by removing isolated nodes (there should not be any)\n",
    "        isolated_nodes = list(nx.isolates(self.network.graph))\n",
    "        if len(isolated_nodes) > 0:\n",
    "            print(\"After removing duplicate edges, some isolated nodes are still present. Removing...\")\n",
    "            self.network.graph.remove_nodes_from(isolated_nodes)\n",
    "                \n",
    "    def convert(\n",
    "            self, minimum_id: int, maximum_id: int, \n",
    "            reverse: bool = True, rename_nodes: bool = True, \n",
    "            remove_duplicates: bool = True,\n",
    "            add_upstream: bool = True, add_downstream: bool = True) -> DamNetwork:\n",
    "        \"\"\"Create a line graph of the original graph such that nodes become edges and edges become nodes\"\"\"\n",
    "        self._add_dummy_edges(minimum_id, maximum_id, add_upstream, add_downstream)\n",
    "        self.network.graph = nx.line_graph(self.network.graph)\n",
    "        for edge in self.network.graph.edges():\n",
    "            upstream, downstream = edge\n",
    "            label = upstream[1]\n",
    "            self.network.graph[edge[0]][edge[1]]['dam_id'] = label  \n",
    "        if rename_nodes:\n",
    "            if self.verbose:\n",
    "                print(\"Renaming nodes..\")\n",
    "            self.rename_nodes()\n",
    "        if remove_duplicates:\n",
    "            if self.verbose:\n",
    "                print(\"Removing duplicates..\")\n",
    "            self._remove_duplicates()\n",
    "        if reverse:\n",
    "            if self.verbose:\n",
    "                print(\"Reversing graph..\")\n",
    "            self._reverse_graph()\n",
    "        return self.network\n",
    "    \n",
    "    def rename_nodes(self, inplace: bool = True) -> nx.DiGraph:\n",
    "        \"\"\"Assigns integer values in a ascending order starting from root node. Requires that the graph\n",
    "        contains only one root node\"\"\"\n",
    "        root_nodes = self.network.root_nodes\n",
    "        if len(root_nodes) > 1:\n",
    "            raise ValueError(\n",
    "                f\"Graph may contain only one root node, {len(root_nodes)} root nodes found\")\n",
    "        # Perform a depth-first traversal starting from the root node(s)\n",
    "        dfs_nodes = list(nx.dfs_preorder_nodes(self.network.graph, source=root_nodes[0]))\n",
    "        mapping = {old_label: new_label for new_label, old_label in enumerate(dfs_nodes)}\n",
    "        renamed_graph = nx.relabel_nodes(self.network.graph, mapping)\n",
    "        if inplace:\n",
    "            self.network.graph = renamed_graph\n",
    "        return renamed_graph"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "656e636b",
   "metadata": {},
   "source": [
    "### Simplify plot, combine and transform Myanmar dam networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03cb0970",
   "metadata": {},
   "outputs": [],
   "source": [
    "salween_network = DamNetwork.from_edges(\n",
    "    salween_edges, coordinates=salween_coordinates)\n",
    "sittaung_network = DamNetwork.from_edges(\n",
    "    sittaung_edges, coordinates=sittaung_coordinates)\n",
    "irrawaddy_network = DamNetwork.from_edges(\n",
    "    irrawaddy_edges, coordinates=irrawaddy_coordinates)\n",
    "# List of dams (reservoirs to retain)\n",
    "retained_node_list = list(pywr_ifc_map.keys())\n",
    "# Outlet nodes\n",
    "out_nodes_sal = {\"outflow_Salween\": 1001}\n",
    "out_nodes_irr = {\"Irrawaddy_Delta_output2\": 1002}\n",
    "out_nodes_sit = {\n",
    "    \"output_Rakhine_1\": 1003, \"output_Rakhine_2\": 1004,\n",
    "    \"output_Rakhine_3\": 1005, \"output_Rakhine_4\": 1006,\n",
    "    \"output_Rakhine_5\": 1007, \"output_Mekong\": 1008,\n",
    "    \"output_Kok\" : 1009, \"output_Sittaung_1\": 1010,\n",
    "    \"output_Sittaung_2\" : 1011, \"output_Sittaung_3\" : 1012,\n",
    "    \"output_Tanintharyi_1\" : 1013, \"output_Tanintharyi_2\" : 1014\n",
    "}\n",
    "\n",
    "for outlet_nodes in (out_nodes_sal, out_nodes_irr, out_nodes_sit):\n",
    "    retained_node_list.extend(list(outlet_nodes.keys()))\n",
    "number_out_nodes = len(out_nodes_sal) + len(out_nodes_irr) + len(out_nodes_sit)\n",
    "print(\n",
    "    f\"Retaining {len(retained_node_list)} nodes, incl. {number_out_nodes}\" +\n",
    "    f\" output nodes and {len(retained_node_list)-number_out_nodes} dams\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3cd5fb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simplify the dam networks\n",
    "n_simplifier_1 = NetworkSimplifier(salween_network, logging=False)\n",
    "n_simplifier_2 = NetworkSimplifier(sittaung_network, logging=False)\n",
    "n_simplifier_3 = NetworkSimplifier(irrawaddy_network, logging=False)\n",
    "# Simplify Salween\n",
    "print(\"Simplifying the Salween model...\")\n",
    "n_simplifier_1.simplify(lambda node : node not in retained_node_list, inplace=True)\n",
    "n_simplifier_1.rename_nodes((pywr_ifc_map, ifc_name_ifc_id_map, out_nodes_sal), inplace=True)\n",
    "n_simplifier_1.reverse_graph(inplace=True)\n",
    "# Simplify Sittaung\n",
    "print(\"Simplifying the Sittaung model...\")\n",
    "n_simplifier_2.simplify(lambda node : node not in retained_node_list, inplace=True)\n",
    "n_simplifier_2.rename_nodes((pywr_ifc_map, ifc_name_ifc_id_map, out_nodes_sit), inplace=True)\n",
    "n_simplifier_2.reverse_graph(inplace=True)\n",
    "# Simplify Irrawaddy\n",
    "print(\"Simplifying the Irrawaddy model...\")\n",
    "n_simplifier_3.simplify(lambda node : node not in retained_node_list, inplace=True)\n",
    "n_simplifier_3.rename_nodes((pywr_ifc_map, ifc_name_ifc_id_map, out_nodes_irr), inplace=True)\n",
    "n_simplifier_3.reverse_graph(inplace=True)\n",
    "print(\"Salween simplified graph\")\n",
    "n_simplifier_1.network.plot(font_size=10, figsize=(5,4))\n",
    "print(\"Sittaung simplified graph\")\n",
    "n_simplifier_2.network.plot(font_size=10, figsize=(5,4))\n",
    "print(\"Irrawaddy simplified graph\")\n",
    "n_simplifier_3.network.plot(font_size=10, figsize=(5,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2add793a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine the three networks into one network sharing the same root node\n",
    "# In this network the nodes are reservoirs defined by their IFC IDs and\n",
    "# the edges represent connections between the reservoirs that are defined by the\n",
    "# river network\n",
    "combined_network = graphtools.combine_multiple_by_root_nodes(\n",
    "    (n_simplifier_1.network, n_simplifier_2.network, n_simplifier_3.network))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "908e0c0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the combined network\n",
    "test1 = TestMOOInputData(\n",
    "    network=combined_network, \n",
    "    dam_data_file= pathlib.Path(\"outputs/moo/all_hp.csv\"))\n",
    "print(\"Testing the combined network\")\n",
    "print(f\"Num dams: {test1.true_data_num_dams}\")\n",
    "test1.test_dams_in_graph()\n",
    "combined_network.plot(font_size = 8, figsize=(7,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "087c5222",
   "metadata": {},
   "outputs": [],
   "source": [
    "converter = NodeToEdgeConverter(combined_network)\n",
    "converter.convert(\n",
    "    2000, 5000, reverse=False, remove_duplicates = False, rename_nodes = False, \n",
    "    add_upstream=False)\n",
    "# Plot the combined disjoint graph\n",
    "converter.network.plot(edge_data_field = 'dam_id', use_coords = False)\n",
    "n_comb = graphtools.combine_disjoint_by_roots(converter.network, inplace=False)\n",
    "# Plot the combined joined graph\n",
    "n_comb.plot(edge_data_field = 'dam_id', use_coords = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46c9d045",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the edge-centric disjoint network (before merging)\n",
    "test2 = TestMOOInputData(network=converter.network)\n",
    "print(\"Testing the disjoint edge-centric network representation\")\n",
    "test2.test_dams_in_graph(graph_type = \"edge-centric\")\n",
    "print(len(test2.network.unique_ids()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b4aa3bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "test3 = TestMOOInputData(network=n_comb)\n",
    "print(\"Testing the combined edge-centric network representation\")\n",
    "test3.test_dams_in_graph(graph_type = \"edge-centric\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1e9a7c9",
   "metadata": {},
   "source": [
    "## NOTE: Set `overwrite_n_comb` to True to overwrite the network file in `n_comb.pickle`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "507e2947",
   "metadata": {},
   "outputs": [],
   "source": [
    "overwrite_n_comb: bool = False\n",
    "if overwrite_n_comb:\n",
    "    n_comb_copy = copy.deepcopy(n_comb)\n",
    "    with open('n_comb.pickle', 'wb') as file_handle:\n",
    "        pickle.dump(n_comb_copy, file_handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7c7bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "converter2 = NodeToEdgeConverter(n_comb, verbose=True)\n",
    "converter2.rename_nodes()\n",
    "converter2.network.plot(edge_data_field = 'dam_id', use_coords = False)\n",
    "converter2._remove_duplicates()\n",
    "converter2.rename_nodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "251d7fbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "test3 = TestMOOInputData(network=converter2.network)\n",
    "print(\"Testing the dcombined edge-centric network representation\")\n",
    "test3.test_dams_in_graph(graph_type = \"edge-centric\")\n",
    "print(len(test3.network.unique_ids()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf18b1f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the numbers of edges and nodes. Number of edges should be\n",
    "# equal to the number of unique IFC ids in the dam database\n",
    "# Number of nodes should be equal to number of edges + 1\n",
    "n_edges = len(converter2.network.graph.edges)\n",
    "n_nodes = len(converter2.network.graph.nodes)\n",
    "print(f\"Number of edges: {n_edges}, Number of nodes: {n_nodes}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "658a85f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dams: Set[int] = set()\n",
    "for (u,v,data) in converter2.network.graph.edges(data=True):\n",
    "    dams.add(data['dam_id'])\n",
    "print(f\"Number of dams in graph = {len(dams)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff90c1eb",
   "metadata": {},
   "source": [
    "## NOTE: Set `overwrite_n_final` to True to overwrite the final edge-centric network file in `n_final.pickle`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72dff289",
   "metadata": {},
   "outputs": [],
   "source": [
    "overwrite_n_final: bool = False\n",
    "if overwrite_n_final:\n",
    "    n_final = copy.deepcopy(converter2.network)\n",
    "    with open('n_final.pickle', 'wb') as file_handle:\n",
    "        pickle.dump(n_final, file_handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b67f061",
   "metadata": {},
   "source": [
    "## Create input file(s) for the MultiObjective Optimization Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3abf6ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_moo_input(\n",
    "        node_criteria: Tuple[str], dam_criteria: Tuple[str],\n",
    "        network_graph: DamNetwork | nx.DiGraph, \n",
    "        dam_node_data: pd.DataFrame,\n",
    "        output_path: pathlib.Path,\n",
    "        id_ifc_map_path: pathlib.Path,\n",
    "        force_status_value: int | None = None) -> nx.DiGraph:\n",
    "    \"\"\"Create an input text file to the multiobjective optimization algorithm\"\"\"\n",
    "    \n",
    "    output_path = pathlib.Path(output_path)\n",
    "    n_node_criteria, n_dam_criteria = len(node_criteria), len(dam_criteria)\n",
    "    \n",
    "    # Mapping between MOO objectives and column names in the dam_node_data dataframe\n",
    "    _criteria_map = {\n",
    "        \"energy\" : \"HP_mean\",\n",
    "        \"ghg\" : \"tot_em\", \n",
    "        \"status\" : \"status_int\", \n",
    "        \"firm_energy\" : \"HP_firm\", \n",
    "        \"loss_agri\" : \"crop_area_loss_km2\", \n",
    "        \"loss_forest\" : \"forest_area_loss_km2\"}\n",
    "    \n",
    "    # Currently connectivity is ignored and therefore, all values have been set to zero\n",
    "    _node_criteria_name_value_map = {\n",
    "        \"connectivity\" : 0\n",
    "    }\n",
    "    \n",
    "    # Get a list of column names to retrieve from dam_node_data dataframe\n",
    "    dam_criteria_cols: List[str] = [_criteria_map.get(name, name) for name in dam_criteria]\n",
    "    \n",
    "    network_graph = copy.deepcopy(network_graph)\n",
    "    \n",
    "    if isinstance(network_graph, DamNetwork):\n",
    "        graph = network_graph.graph\n",
    "    else:\n",
    "        graph = network_graph\n",
    "\n",
    "    # Rename nodes such that the nodes are named with integers from 0 to N-1 where N is the number of nodes\n",
    "    graph = nx.relabel.convert_node_labels_to_integers(\n",
    "        graph, first_label=0, ordering='default')\n",
    "    \n",
    "    root_nodes = [node for node in graph.nodes if graph.in_degree(node) == 0]\n",
    "    if len(root_nodes) > 1:\n",
    "        raise ValueError(\"Graph has more than one root node\")\n",
    "    else:\n",
    "        root_node = root_nodes[0]\n",
    "    \n",
    "    # Create templates for individual sections of the text file    \n",
    "    header_line_1_template = string.Template(\n",
    "        \"p ${n_nodes} ${n_edges} ${n_node_criteria} ${n_dam_criteria}\")\n",
    "    header_line_2_template = string.Template(\"dam_criteria ${dam_criteria_str}\")\n",
    "    header_line_3_template = string.Template(\"node_criteria ${node_criteria_str}\")\n",
    "    dam_line_template = string.Template(\"d ${dam_id} ${dam_criteria_values_str}\")\n",
    "    node_line_template = string.Template(\"n ${node_id} ${node_criteria_values_str}\")\n",
    "    root_node_line_template = string.Template(\"r ${root_node_id}\")\n",
    "    edge_line_template = string.Template(\"e ${up_node_id} ${down_node_id} ${dam_id}\")\n",
    "\n",
    "    dams: Set[int] = set()\n",
    "    for (u,v,data) in graph.edges(data=True):\n",
    "        dams.add(data['dam_id'])\n",
    "    print(f\"Number of dams after in graph after renaming nodes = {len(dams)}\")\n",
    "    \n",
    "    # Enquire the graph object to get information about its nodes (river sections) and edges (dams)\n",
    "    # Get nodes in topological order\n",
    "    nodes = graph.nodes\n",
    "    edges = graph.edges(data=True)\n",
    "    n_nodes, n_edges = len(nodes), len(edges)\n",
    "    print(f\"Processing graph with {n_nodes} nodes and {n_edges} edges.\")\n",
    "    dam_criteria_str = \" \".join(dam_criteria)\n",
    "    node_criteria_str = \" \".join(node_criteria)\n",
    "    print(\"Writing header lines\")\n",
    "    strings: List[str] = []\n",
    "    strings.append(header_line_1_template.safe_substitute(\n",
    "        n_nodes=n_nodes, n_edges=n_edges, n_node_criteria=n_node_criteria, \n",
    "        n_dam_criteria=n_dam_criteria))\n",
    "    strings.append(header_line_2_template.safe_substitute(\n",
    "        dam_criteria_str=dam_criteria_str))\n",
    "    if len(node_criteria) > 0:\n",
    "        strings.append(header_line_3_template.safe_substitute(\n",
    "            node_criteria_str=node_criteria_str))\n",
    "    \n",
    "    # Add dams\n",
    "    # The code requires dams to be listed in ascending order with indices 1 to N\n",
    "    # For this reason we need to rename the edges first and then list them in ascending order\n",
    "    # betore writing them to file\n",
    "    id_to_ifc = dict()\n",
    "    old_edges =  sorted(edges, key=lambda edge: edge[2]['dam_id'])\n",
    "    updated_edges = [] # Edges with update IDs\n",
    "    for ix, (u, v, edge_data) in enumerate(old_edges):\n",
    "        ifc_id = edge_data['dam_id']\n",
    "        new_id = ix + 1\n",
    "        new_edge = (u, v, {'dam_id': new_id})\n",
    "        updated_edges.append(new_edge)\n",
    "        # Write entries to a dictionary providing mapping between\n",
    "        # new dam_ids (edge_ids) for algorithmic purposes and the original dam ids\n",
    "        id_to_ifc[new_id] = ifc_id\n",
    "        \n",
    "    print(f\"Saving the new-old dam ID map to {id_ifc_map_path.as_posix()}\")\n",
    "    graphtools.dict_to_json(id_to_ifc, id_ifc_map_path)\n",
    "    \n",
    "    sorted_updated_edges = sorted(\n",
    "        updated_edges, key=lambda edge: edge[2]['dam_id'])\n",
    "    \n",
    "    for edge in sorted_updated_edges:\n",
    "        dam_id = edge[2]['dam_id']\n",
    "        dam_criteria_arr = dam_node_data.loc[id_to_ifc[dam_id], dam_criteria_cols]\n",
    "        if force_status_value is not None:\n",
    "            dam_criteria_arr[_criteria_map[\"status\"]] = force_status_value\n",
    "        dam_criteria_string = np.array2string(\n",
    "            dam_criteria_arr.values, separator=' ').lstrip(\"[\").rstrip(\"]\")\n",
    "        strings.append(dam_line_template.safe_substitute(\n",
    "            dam_id = dam_id, dam_criteria_values_str = dam_criteria_string))\n",
    "    \n",
    "    # Add nodes (WATCH OUT: IF NODE CRITERIA ARE ADDED WE NEED TO MAKE SURE\n",
    "    # THAT NODES CORRESPOND TO (MODIFIED) EDGES. Does not matter at this stage\n",
    "    # as we feel all nodes with connectivity criterion of zero anyway)\n",
    "    for node in sorted(nodes):\n",
    "        _node_criteria_vals = []\n",
    "        for criterion in node_criteria:\n",
    "            _node_criteria_vals.append(_node_criteria_name_value_map.get(criterion, 'NA'))\n",
    "        _node_criteria_vals_str = \" \".join(map(str, _node_criteria_vals))\n",
    "        strings.append(node_line_template.safe_substitute(\n",
    "            node_id = node, node_criteria_values_str = _node_criteria_vals_str))\n",
    "    \n",
    "    # Add root node\n",
    "    strings.append(root_node_line_template.safe_substitute(root_node_id=root_node))\n",
    "    # Add information about the edges using the data stored in the network graph\n",
    "    for edge in sorted_updated_edges:\n",
    "        source_node, target_node, edge_data = edge \n",
    "        dam_id_value = edge_data.get('dam_id', 'unknown')\n",
    "        strings.append(edge_line_template.safe_substitute(\n",
    "            up_node_id = source_node,\n",
    "            down_node_id = target_node,\n",
    "            dam_id = dam_id_value))\n",
    "    \n",
    "    with open(output_path, 'w') as file:\n",
    "        for line in strings:\n",
    "            file.write(line + '\\n')\n",
    "\n",
    "    return network_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa310fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: the data contains rows with repeated rows (for some reason). The rows contain the same information\n",
    "#       We therefore remove the redundant rows and keep only the first row\n",
    "dam_node_data = pd.read_csv(ifc_id_map_path, index_col=[0]).set_index('ifc_id')\n",
    "dam_node_data = dam_node_data[~dam_node_data.index.duplicated(keep='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69c251d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "dam_node_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f14b3a5",
   "metadata": {},
   "source": [
    "### Create input files with multiple options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6b67d0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_network = converter2.network\n",
    "# 5 objectives built dam statuses included in the analysis\n",
    "create_moo_input( \n",
    "    node_criteria = ('connectivity', ), \n",
    "    dam_criteria = (\"energy\", \"ghg\", \"status\", \"firm_energy\", \"loss_agri\", \"loss_forest\"),\n",
    "    network_graph = final_network.graph,\n",
    "    dam_node_data = dam_node_data,\n",
    "    id_ifc_map_path=pathlib.Path(\"outputs/moo/id_to_ifc.json\"),\n",
    "    force_status_value = None,\n",
    "    output_path = pathlib.Path(\"moo_solver/solutions_and_inputs/mya_5_obj_built.txt\"))\n",
    "# 5 objectives no built dams\n",
    "create_moo_input( \n",
    "    node_criteria = ('connectivity', ), \n",
    "    dam_criteria = (\"energy\", \"ghg\", \"status\", \"firm_energy\", \"loss_agri\", \"loss_forest\"),\n",
    "    network_graph = final_network.graph,\n",
    "    dam_node_data = dam_node_data,\n",
    "    id_ifc_map_path=pathlib.Path(\"outputs/moo/id_to_ifc.json\"),\n",
    "    force_status_value = 0,\n",
    "    output_path = pathlib.Path(\"moo_solver/solutions_and_inputs/mya_5_obj_nobuilt.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a8efc26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5 objectives built dam statuses included in the analysis\n",
    "create_moo_input( \n",
    "    node_criteria = ('connectivity', ), \n",
    "    dam_criteria = (\"energy\", \"ghg\", \"status\", \"firm_energy\", \"loss_agri\", \"loss_forest\"),\n",
    "    network_graph = final_network.graph,\n",
    "    dam_node_data = dam_node_data,\n",
    "    id_ifc_map_path=pathlib.Path(\"outputs/moo/id_to_ifc.json\"),\n",
    "    force_status_value = None,\n",
    "    output_path = pathlib.Path(\"moo_solver_CPAIOR/Basin_Input_Files/mya_5_obj_built.txt\"))\n",
    "# 5 objectives no built dams\n",
    "create_moo_input( \n",
    "    node_criteria = ('connectivity', ), \n",
    "    dam_criteria = (\"energy\", \"ghg\", \"status\", \"firm_energy\", \"loss_agri\", \"loss_forest\"),\n",
    "    network_graph = final_network.graph,\n",
    "    dam_node_data = dam_node_data,\n",
    "    id_ifc_map_path=pathlib.Path(\"outputs/moo/id_to_ifc.json\"),\n",
    "    force_status_value = 0,\n",
    "    output_path = pathlib.Path(\"moo_solver_CPAIOR/Basin_Input_Files/mya_5_obj_nobuilt.txt\"))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6dc5929f",
   "metadata": {},
   "source": [
    "# 5 objectives all dams built\n",
    "create_moo_input( \n",
    "    node_criteria = ('connectivity', ), \n",
    "    dam_criteria = (\"energy\", \"ghg\", \"status\", \"firm_energy\", \"loss_agri\", \"loss_forest\"),\n",
    "    network_graph = final_network.graph,\n",
    "    dam_node_data = dam_node_data,\n",
    "    force_status_value = 1,\n",
    "    output_path = pathlib.Path(\"moo_solver/solutions_and_inputs/mya_5_obj_allbuilt.txt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a30152b",
   "metadata": {},
   "source": [
    "## Tests on simple graph representations"
   ]
  },
  {
   "cell_type": "raw",
   "id": "7b4cf79e",
   "metadata": {},
   "source": [
    "edges = [(1, 2), (2, 3), (4, 3), (3, 5)]\n",
    "# Create a directed graph from the list of edges\n",
    "G = nx.DiGraph(edges)\n",
    "ex_network = DamNetwork(copy.deepcopy(G))\n",
    "ex_network.plot(edge_data_field = 'dam_id', figsize=(4,3))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "673ac70f",
   "metadata": {},
   "source": [
    "converter = NodeToEdgeConverter(ex_network)\n",
    "converter._reverse_graph()\n",
    "converter.convert(666, 777, reverse=False, remove_duplicates = True)\n",
    "converter.rename_nodes()\n",
    "print(converter.edge_data)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "90f81b66",
   "metadata": {},
   "source": [
    "ex_network.plot(edge_data_field = 'dam_id', figsize=(4,3))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}