Dependency_Graphs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Identifying the level of each housing characteristic\n",
    "\n",
    "This notebook visualizes the ResStock project as a hierarchical graph with the nodes being the housing characteristics the dependencies as directed edges.  The housing characteristics at the top of the graph are high level characteristics that have little dependencies, where housing characteristics at the bottom of the graph have many dependencies or depend on characteristics that have many dependencies.\n",
    "\n",
    "## Python Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7.4 (default, Aug 13 2019, 15:17:50) \n",
      "[Clang 4.0.1 (tags/RELEASE_401/final)]\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print(sys.version)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "import networkx as nx\n",
    "from nxpd import draw\n",
    "from os.path import isfile, join\n",
    "from IPython.display import display\n",
    "from networkx.drawing.nx_pydot import graphviz_layout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_adj_graph(path_adj):\n",
    "    \"\"\"\n",
    "    This function loads the adjacency matrix in as a data frame and then converts\n",
    "    it to a graph.\n",
    "    \"\"\"\n",
    "    adj_df = pd.read_csv(path_adj,index_col=0)\n",
    "\n",
    "    # Create Graph\n",
    "    G = nx.from_pandas_adjacency(adj_df,nx.DiGraph())\n",
    "\n",
    "    # Reset index for convienience\n",
    "    adj_df.reset_index(inplace=True,drop=True)\n",
    "    \n",
    "    return adj_df,G\n",
    "\n",
    "def get_level_df(adj_df,longest_path_len,n_nodes):\n",
    "    \"\"\"\n",
    "    This function returns the level of each housing characteristic(0 is no dependencies)\n",
    "    \"\"\"\n",
    "    # Initialize node level values\n",
    "    level = np.zeros_like(adj_df[adj_df.columns[0]])\n",
    "\n",
    "    # Iterate over nodes up to the longest path length\n",
    "    for j in range(int(longest_path_len)):\n",
    "        # For each housing characteristic node\n",
    "        for i in range(n_nodes):\n",
    "            # Get the index of the dependencies\n",
    "            column_name = adj_df.columns[i]\n",
    "            dependencies = np.where(adj_df[column_name] == 1)[0]\n",
    "\n",
    "            #If there are dependencies\n",
    "            if len(dependencies) > 0:\n",
    "                # Get the level of the dependencies\n",
    "                dependency_levels = level[dependencies]\n",
    "\n",
    "                # Identify the maximum level of the dependencies\n",
    "                max_level = np.max(dependency_levels)\n",
    "\n",
    "                # This node's level must be one greater than the max level\n",
    "                level[i] = max_level + 1\n",
    "\n",
    "    # Print housing characteristics grouped by levels\n",
    "    hc_level_names = list()\n",
    "    levels = list()\n",
    "    for i in range(int(longest_path_len)):\n",
    "        idx = np.where(level == i)[0]\n",
    "        for j in range(len(idx)):\n",
    "            hc_level_names.append(node_names[idx[j]])\n",
    "            levels.append(i)\n",
    "    \n",
    "    # Create Data Frame\n",
    "    hc_level_df = pd.DataFrame(columns=['housing_characteristic','level'])\n",
    "    hc_level_df['housing_characteristic'] = hc_level_names\n",
    "    hc_level_df['level'] = levels\n",
    "    \n",
    "    return hc_level_df,level\n",
    "\n",
    "def isolate_levels(G,level,node_names):\n",
    "    \"\"\"\n",
    "    This function gets edges that do not jump different levels.\n",
    "    \"\"\"\n",
    "    jump_edges = list()\n",
    "    edges_new = list()\n",
    "    for edge in G.edges:\n",
    "        idx_in = np.where(edge[0] == node_names)[0][0]\n",
    "        idx_out = np.where(edge[1] == node_names)[0][0]\n",
    "\n",
    "        jump = int(np.fabs(level[idx_in] - level[idx_out]))\n",
    "        if jump > 1:\n",
    "            jump_edges.append(edge)\n",
    "        else:\n",
    "            edges_new.append(edge)\n",
    "\n",
    "    return edges_new\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating a dependency graphs for project:\n",
      "----------------------------------------\n",
      "hvac_characteristics_new/visualizations/adjacency_matrix.csv\n",
      "hvac_characteristics_new\n",
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 13\n",
      "Number of edges: 18\n",
      "Average in degree:   1.3846\n",
      "Average out degree:   1.3846\n",
      "Longest path length:\t 4\n",
      "Maximum levels: \t 3\n",
      "hvac_characteristics_old/visualizations/adjacency_matrix.csv\n",
      "hvac_characteristics_old\n",
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 23\n",
      "Number of edges: 75\n",
      "Average in degree:   3.2609\n",
      "Average out degree:   3.2609\n",
      "Longest path length:\t 4\n",
      "Maximum levels: \t 3\n"
     ]
    }
   ],
   "source": [
    "# Get all project directories\n",
    "project_dirs = glob(join('hvac_*'))\n",
    "\n",
    "print('Creating a dependency graphs for project:')\n",
    "print('----------------------------------------')\n",
    "for project_dir in project_dirs:\n",
    "\n",
    "    # Create path for adjacency matrix\n",
    "    path_adj = join(project_dir,'visualizations/adjacency_matrix.csv')\n",
    "    print(path_adj)\n",
    "\n",
    "    # Load adjacency matric and construct graph\n",
    "    try:\n",
    "        # Load adjacency matrix and graph\n",
    "        adj_df,G = load_adj_graph(path_adj)\n",
    "\n",
    "        # Get information about the graph\n",
    "        print(project_dir)\n",
    "        n_nodes = len(G.nodes)\n",
    "        n_edges = len(G.edges)\n",
    "        node_names = adj_df.columns.values\n",
    "        longest_path_len = nx.dag_longest_path_length(G)\n",
    "        edges_org = G.edges\n",
    "        print(nx.info(G))\n",
    "        print('Longest path length:\\t %d' % longest_path_len)\n",
    "        \n",
    "        # Get levels\n",
    "        hc_level_df,level = get_level_df(adj_df,longest_path_len,n_nodes)\n",
    "        print('Maximum levels: \\t %d' % hc_level_df['level'].max() )\n",
    "        print\n",
    "        \n",
    "        # Save the levels data frame\n",
    "        ## Make directory if it doesn't exist\n",
    "        outdir = join(project_dir,'visualizations')\n",
    "        if not os.path.exists(outdir):\n",
    "            os.mkdir(outdir)\n",
    "\n",
    "        ## Save data frame\n",
    "        hc_level_df.to_csv(join(outdir,'hc_levels.csv'),index=False)\n",
    "        \n",
    "        edges_new = isolate_levels(G,level,node_names)\n",
    "        \n",
    "        # Create Plots\n",
    "        G_new = nx.DiGraph()\n",
    "        G_new.add_nodes_from(node_names)\n",
    "        G_new.add_edges_from(edges_new)\n",
    "\n",
    "        scale_x = 1.0\n",
    "        scale_y = 1.0\n",
    "        pos = {}\n",
    "        for i in range(int(longest_path_len)+1):\n",
    "            idx = np.where(level == i)[0][::-1]\n",
    "            n_level = len(idx)\n",
    "\n",
    "            j = 1\n",
    "            for idx_i in idx:\n",
    "                y = n_level*scale_y*(float(j)/float(n_level)) - n_level/2.0\n",
    "                x = i*scale_x\n",
    "                pos[node_names[idx_i]] = (x,y)\n",
    "                j += 1\n",
    "\n",
    "        out_file = join(outdir,'Dependency_Graph_Full.pdf')\n",
    "        draw(G,filename=out_file,show=False) # Full Graph\n",
    "        out_file = join(outdir,'Dependency_Graph_Levels.pdf')\n",
    "        draw(G_new,filename=out_file,show=False) # Levels Graph\n",
    "    except IOError:\n",
    "        pass\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}