AnneFrankDiaryAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anne Frank Diary Analysis\n",
    "## By Mark McGown and Moses Boudourides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Jwb1Nwb59eXt",
    "outputId": "5381fabb-87c8-490b-ff0e-cf710698080a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n",
      "Collecting leidenalg\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b6/cc/d76baf78a3924ba6093a3ce8d14e2289f1d718bd3bcbb8252bb131d12daa/leidenalg-0.7.0.tar.gz (92kB)\n",
      "\u001b[K     |████████████████████████████████| 102kB 2.6MB/s \n",
      "\u001b[?25hCollecting python-igraph>=0.7.1.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f3/23/2959ac50ac7a3d8c28602a752075abd21025767fc32d4587fb35ae273d22/python_igraph-0.8.0-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n",
      "\u001b[K     |████████████████████████████████| 3.2MB 8.7MB/s \n",
      "\u001b[?25hCollecting texttable>=1.6.2\n",
      "  Downloading https://files.pythonhosted.org/packages/ec/b1/8a1c659ce288bf771d5b1c7cae318ada466f73bd0e16df8d86f27a2a3ee7/texttable-1.6.2-py2.py3-none-any.whl\n",
      "Building wheels for collected packages: leidenalg\n",
      "  Building wheel for leidenalg (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for leidenalg: filename=leidenalg-0.7.0-cp36-cp36m-linux_x86_64.whl size=1107506 sha256=47ecfc5e3c941152bdb3aacde7179c2db386c0ee06ca112913c51858f7393550\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/55/48/5a04693a10f50297bcda23819ca23ab3470a61dd911851c8bd\n",
      "Successfully built leidenalg\n",
      "Installing collected packages: texttable, python-igraph, leidenalg\n",
      "Successfully installed leidenalg-0.7.0 python-igraph-0.8.0 texttable-1.6.2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "pkg-config is already the newest version (0.29.1-0ubuntu2).\n",
      "graphviz is already the newest version (2.40.1-2).\n",
      "The following NEW packages will be installed:\n",
      "  libgraphviz-dev libgvc6-plugins-gtk libxdot4\n",
      "0 upgraded, 3 newly installed, 0 to remove and 25 not upgraded.\n",
      "Need to get 91.3 kB of archives.\n",
      "After this operation, 425 kB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libxdot4 amd64 2.40.1-2 [15.7 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libgvc6-plugins-gtk amd64 2.40.1-2 [18.2 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libgraphviz-dev amd64 2.40.1-2 [57.3 kB]\n",
      "Fetched 91.3 kB in 1s (149 kB/s)\n",
      "debconf: unable to initialize frontend: Dialog\n",
      "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 3.)\n",
      "debconf: falling back to frontend: Readline\n",
      "debconf: unable to initialize frontend: Readline\n",
      "debconf: (This frontend requires a controlling tty.)\n",
      "debconf: falling back to frontend: Teletype\n",
      "dpkg-preconfigure: unable to re-open stdin: \n",
      "Selecting previously unselected package libxdot4.\n",
      "(Reading database ... 133872 files and directories currently installed.)\n",
      "Preparing to unpack .../libxdot4_2.40.1-2_amd64.deb ...\n",
      "Unpacking libxdot4 (2.40.1-2) ...\n",
      "Selecting previously unselected package libgvc6-plugins-gtk.\n",
      "Preparing to unpack .../libgvc6-plugins-gtk_2.40.1-2_amd64.deb ...\n",
      "Unpacking libgvc6-plugins-gtk (2.40.1-2) ...\n",
      "Selecting previously unselected package libgraphviz-dev.\n",
      "Preparing to unpack .../libgraphviz-dev_2.40.1-2_amd64.deb ...\n",
      "Unpacking libgraphviz-dev (2.40.1-2) ...\n",
      "Setting up libxdot4 (2.40.1-2) ...\n",
      "Setting up libgvc6-plugins-gtk (2.40.1-2) ...\n",
      "Setting up libgraphviz-dev (2.40.1-2) ...\n",
      "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
      "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
      "\n",
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "The following additional packages will be installed:\n",
      "  libpython-all-dev python-all python-all-dev python-asn1crypto\n",
      "  python-cffi-backend python-crypto python-cryptography python-dbus\n",
      "  python-enum34 python-gi python-idna python-ipaddress python-keyring\n",
      "  python-keyrings.alt python-pip-whl python-pkg-resources python-secretstorage\n",
      "  python-setuptools python-six python-wheel python-xdg python3-pkg-resources\n",
      "  python3-virtualenv virtualenv\n",
      "Suggested packages:\n",
      "  python-crypto-doc python-cryptography-doc python-cryptography-vectors\n",
      "  python-dbus-dbg python-dbus-doc python-enum34-doc python-gi-cairo\n",
      "  gnome-keyring libkf5wallet-bin gir1.2-gnomekeyring-1.0 python-fs\n",
      "  python-gdata python-keyczar python-secretstorage-doc python-setuptools-doc\n",
      "  python3-setuptools\n",
      "The following NEW packages will be installed:\n",
      "  libpython-all-dev python-all python-all-dev python-asn1crypto\n",
      "  python-cffi-backend python-crypto python-cryptography python-dbus\n",
      "  python-enum34 python-gi python-idna python-ipaddress python-keyring\n",
      "  python-keyrings.alt python-pip python-pip-whl python-pkg-resources\n",
      "  python-secretstorage python-setuptools python-six python-virtualenv\n",
      "  python-wheel python-xdg python3-pkg-resources python3-virtualenv virtualenv\n",
      "0 upgraded, 26 newly installed, 0 to remove and 25 not upgraded.\n",
      "Need to get 3,569 kB of archives.\n",
      "After this operation, 11.4 MB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libpython-all-dev amd64 2.7.15~rc1-1 [1,092 B]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-all amd64 2.7.15~rc1-1 [1,076 B]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-all-dev amd64 2.7.15~rc1-1 [1,100 B]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-asn1crypto all 0.24.0-1 [72.7 kB]\n",
      "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-cffi-backend amd64 1.11.5-1 [63.4 kB]\n",
      "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-crypto amd64 2.6.1-8ubuntu2 [244 kB]\n",
      "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-enum34 all 1.1.6-2 [34.8 kB]\n",
      "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-idna all 2.6-1 [32.4 kB]\n",
      "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-ipaddress all 1.0.17-1 [18.2 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-six all 1.11.0-2 [11.3 kB]\n",
      "Get:11 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python-cryptography amd64 2.1.4-1ubuntu1.3 [221 kB]\n",
      "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-dbus amd64 1.2.6-1 [90.2 kB]\n",
      "Get:13 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python-gi amd64 3.26.1-2ubuntu1 [197 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-secretstorage all 2.3.1-2 [11.8 kB]\n",
      "Get:15 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-keyring all 10.6.0-1 [30.6 kB]\n",
      "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-keyrings.alt all 3.0-1 [16.7 kB]\n",
      "Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-pip-whl all 9.0.1-2.3~ubuntu1.18.04.1 [1,653 kB]\n",
      "Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-pip all 9.0.1-2.3~ubuntu1.18.04.1 [151 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-pkg-resources all 39.0.1-2 [128 kB]\n",
      "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 python-setuptools all 39.0.1-2 [329 kB]\n",
      "Get:21 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-virtualenv all 15.1.0+ds-1.1 [46.8 kB]\n",
      "Get:22 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-wheel all 0.30.0-0.2 [36.4 kB]\n",
      "Get:23 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-xdg all 0.25-4ubuntu1 [31.3 kB]\n",
      "Get:24 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pkg-resources all 39.0.1-2 [98.8 kB]\n",
      "Get:25 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-virtualenv all 15.1.0+ds-1.1 [43.4 kB]\n",
      "Get:26 http://archive.ubuntu.com/ubuntu bionic/universe amd64 virtualenv all 15.1.0+ds-1.1 [4,476 B]\n",
      "Fetched 3,569 kB in 1s (3,408 kB/s)\n",
      "debconf: unable to initialize frontend: Dialog\n",
      "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 26.)\n",
      "debconf: falling back to frontend: Readline\n",
      "debconf: unable to initialize frontend: Readline\n",
      "debconf: (This frontend requires a controlling tty.)\n",
      "debconf: falling back to frontend: Teletype\n",
      "dpkg-preconfigure: unable to re-open stdin: \n",
      "Selecting previously unselected package libpython-all-dev:amd64.\n",
      "(Reading database ... 133947 files and directories currently installed.)\n",
      "Preparing to unpack .../00-libpython-all-dev_2.7.15~rc1-1_amd64.deb ...\n",
      "Unpacking libpython-all-dev:amd64 (2.7.15~rc1-1) ...\n",
      "Selecting previously unselected package python-all.\n",
      "Preparing to unpack .../01-python-all_2.7.15~rc1-1_amd64.deb ...\n",
      "Unpacking python-all (2.7.15~rc1-1) ...\n",
      "Selecting previously unselected package python-all-dev.\n",
      "Preparing to unpack .../02-python-all-dev_2.7.15~rc1-1_amd64.deb ...\n",
      "Unpacking python-all-dev (2.7.15~rc1-1) ...\n",
      "Selecting previously unselected package python-asn1crypto.\n",
      "Preparing to unpack .../03-python-asn1crypto_0.24.0-1_all.deb ...\n",
      "Unpacking python-asn1crypto (0.24.0-1) ...\n",
      "Selecting previously unselected package python-cffi-backend.\n",
      "Preparing to unpack .../04-python-cffi-backend_1.11.5-1_amd64.deb ...\n",
      "Unpacking python-cffi-backend (1.11.5-1) ...\n",
      "Selecting previously unselected package python-crypto.\n",
      "Preparing to unpack .../05-python-crypto_2.6.1-8ubuntu2_amd64.deb ...\n",
      "Unpacking python-crypto (2.6.1-8ubuntu2) ...\n",
      "Selecting previously unselected package python-enum34.\n",
      "Preparing to unpack .../06-python-enum34_1.1.6-2_all.deb ...\n",
      "Unpacking python-enum34 (1.1.6-2) ...\n",
      "Selecting previously unselected package python-idna.\n",
      "Preparing to unpack .../07-python-idna_2.6-1_all.deb ...\n",
      "Unpacking python-idna (2.6-1) ...\n",
      "Selecting previously unselected package python-ipaddress.\n",
      "Preparing to unpack .../08-python-ipaddress_1.0.17-1_all.deb ...\n",
      "Unpacking python-ipaddress (1.0.17-1) ...\n",
      "Selecting previously unselected package python-six.\n",
      "Preparing to unpack .../09-python-six_1.11.0-2_all.deb ...\n",
      "Unpacking python-six (1.11.0-2) ...\n",
      "Selecting previously unselected package python-cryptography.\n",
      "Preparing to unpack .../10-python-cryptography_2.1.4-1ubuntu1.3_amd64.deb ...\n",
      "Unpacking python-cryptography (2.1.4-1ubuntu1.3) ...\n",
      "Selecting previously unselected package python-dbus.\n",
      "Preparing to unpack .../11-python-dbus_1.2.6-1_amd64.deb ...\n",
      "Unpacking python-dbus (1.2.6-1) ...\n",
      "Selecting previously unselected package python-gi.\n",
      "Preparing to unpack .../12-python-gi_3.26.1-2ubuntu1_amd64.deb ...\n",
      "Unpacking python-gi (3.26.1-2ubuntu1) ...\n",
      "Selecting previously unselected package python-secretstorage.\n",
      "Preparing to unpack .../13-python-secretstorage_2.3.1-2_all.deb ...\n",
      "Unpacking python-secretstorage (2.3.1-2) ...\n",
      "Selecting previously unselected package python-keyring.\n",
      "Preparing to unpack .../14-python-keyring_10.6.0-1_all.deb ...\n",
      "Unpacking python-keyring (10.6.0-1) ...\n",
      "Selecting previously unselected package python-keyrings.alt.\n",
      "Preparing to unpack .../15-python-keyrings.alt_3.0-1_all.deb ...\n",
      "Unpacking python-keyrings.alt (3.0-1) ...\n",
      "Selecting previously unselected package python-pip-whl.\n",
      "Preparing to unpack .../16-python-pip-whl_9.0.1-2.3~ubuntu1.18.04.1_all.deb ...\n",
      "Unpacking python-pip-whl (9.0.1-2.3~ubuntu1.18.04.1) ...\n",
      "Selecting previously unselected package python-pip.\n",
      "Preparing to unpack .../17-python-pip_9.0.1-2.3~ubuntu1.18.04.1_all.deb ...\n",
      "Unpacking python-pip (9.0.1-2.3~ubuntu1.18.04.1) ...\n",
      "Selecting previously unselected package python-pkg-resources.\n",
      "Preparing to unpack .../18-python-pkg-resources_39.0.1-2_all.deb ...\n",
      "Unpacking python-pkg-resources (39.0.1-2) ...\n",
      "Selecting previously unselected package python-setuptools.\n",
      "Preparing to unpack .../19-python-setuptools_39.0.1-2_all.deb ...\n",
      "Unpacking python-setuptools (39.0.1-2) ...\n",
      "Selecting previously unselected package python-virtualenv.\n",
      "Preparing to unpack .../20-python-virtualenv_15.1.0+ds-1.1_all.deb ...\n",
      "Unpacking python-virtualenv (15.1.0+ds-1.1) ...\n",
      "Selecting previously unselected package python-wheel.\n",
      "Preparing to unpack .../21-python-wheel_0.30.0-0.2_all.deb ...\n",
      "Unpacking python-wheel (0.30.0-0.2) ...\n",
      "Selecting previously unselected package python-xdg.\n",
      "Preparing to unpack .../22-python-xdg_0.25-4ubuntu1_all.deb ...\n",
      "Unpacking python-xdg (0.25-4ubuntu1) ...\n",
      "Selecting previously unselected package python3-pkg-resources.\n",
      "Preparing to unpack .../23-python3-pkg-resources_39.0.1-2_all.deb ...\n",
      "Unpacking python3-pkg-resources (39.0.1-2) ...\n",
      "Selecting previously unselected package python3-virtualenv.\n",
      "Preparing to unpack .../24-python3-virtualenv_15.1.0+ds-1.1_all.deb ...\n",
      "Unpacking python3-virtualenv (15.1.0+ds-1.1) ...\n",
      "Selecting previously unselected package virtualenv.\n",
      "Preparing to unpack .../25-virtualenv_15.1.0+ds-1.1_all.deb ...\n",
      "Unpacking virtualenv (15.1.0+ds-1.1) ...\n",
      "Setting up python-idna (2.6-1) ...\n",
      "Setting up python-pip-whl (9.0.1-2.3~ubuntu1.18.04.1) ...\n",
      "Setting up python-asn1crypto (0.24.0-1) ...\n",
      "Setting up python-crypto (2.6.1-8ubuntu2) ...\n",
      "Setting up python-wheel (0.30.0-0.2) ...\n",
      "Setting up libpython-all-dev:amd64 (2.7.15~rc1-1) ...\n",
      "Setting up python3-pkg-resources (39.0.1-2) ...\n",
      "Setting up python-pkg-resources (39.0.1-2) ...\n",
      "Setting up python-cffi-backend (1.11.5-1) ...\n",
      "Setting up python-gi (3.26.1-2ubuntu1) ...\n",
      "Setting up python-six (1.11.0-2) ...\n",
      "Setting up python3-virtualenv (15.1.0+ds-1.1) ...\n",
      "Setting up python-enum34 (1.1.6-2) ...\n",
      "Setting up python-virtualenv (15.1.0+ds-1.1) ...\n",
      "Setting up python-dbus (1.2.6-1) ...\n",
      "Setting up python-ipaddress (1.0.17-1) ...\n",
      "Setting up python-pip (9.0.1-2.3~ubuntu1.18.04.1) ...\n",
      "Setting up virtualenv (15.1.0+ds-1.1) ...\n",
      "Setting up python-all (2.7.15~rc1-1) ...\n",
      "Setting up python-xdg (0.25-4ubuntu1) ...\n",
      "Setting up python-setuptools (39.0.1-2) ...\n",
      "Setting up python-keyrings.alt (3.0-1) ...\n",
      "Setting up python-all-dev (2.7.15~rc1-1) ...\n",
      "Setting up python-cryptography (2.1.4-1ubuntu1.3) ...\n",
      "Setting up python-secretstorage (2.3.1-2) ...\n",
      "Setting up python-keyring (10.6.0-1) ...\n",
      "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
      "Collecting pygraphviz\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/b1/d6d849ddaf6f11036f9980d433f383d4c13d1ebcfc3cd09bc845bda7e433/pygraphviz-1.5.zip (117kB)\n",
      "\u001b[K     |████████████████████████████████| 122kB 2.8MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: pygraphviz\n",
      "  Building wheel for pygraphviz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pygraphviz: filename=pygraphviz-1.5-cp36-cp36m-linux_x86_64.whl size=157588 sha256=1158b92e1cb6cdc0e82c38a9dd405564243a8a51441aa9c9bd4c30fe2f725830\n",
      "  Stored in directory: /root/.cache/pip/wheels/65/54/69/1aee9e66ab19916293208d4c9de0d3898adebe6b2eeff6476b\n",
      "Successfully built pygraphviz\n",
      "Installing collected packages: pygraphviz\n",
      "Successfully installed pygraphviz-1.5\n",
      "Collecting en_core_web_lg==2.2.5\n",
      "\u001b[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-2.2.5/en_core_web_lg-2.2.5.tar.gz (827.9MB)\n",
      "\u001b[K     |████████████████████████████████| 827.9MB 147.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.6/dist-packages (from en_core_web_lg==2.2.5) (2.2.4)\n",
      "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (7.4.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.18.2)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.1.3)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (0.6.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (4.38.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (2.0.3)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (3.0.2)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (46.0.0)\n",
      "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (2.21.0)\n",
      "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (1.0.2)\n",
      "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_lg==2.2.5) (0.4.1)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_lg==2.2.5) (1.5.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (2019.11.28)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (1.24.3)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_lg==2.2.5) (2.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_lg==2.2.5) (3.1.0)\n",
      "Building wheels for collected packages: en-core-web-lg\n",
      "  Building wheel for en-core-web-lg (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for en-core-web-lg: filename=en_core_web_lg-2.2.5-cp36-none-any.whl size=829180944 sha256=b41fd9b608d43262e3d3c572645ee88be961e9ffcad8d871a6d8504452530993\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-a_h_cgrs/wheels/2a/c1/a6/fc7a877b1efca9bc6a089d6f506f16d3868408f9ff89f8dbfc\n",
      "Successfully built en-core-web-lg\n",
      "Installing collected packages: en-core-web-lg\n",
      "Successfully installed en-core-web-lg-2.2.5\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the model via spacy.load('en_core_web_lg')\n"
     ]
    }
   ],
   "source": [
    "#Source https://archive.org/stream/AnneFrankTheDiaryOfAYoungGirl_201606/Anne-Frank-The-Diary-Of-A-Young-Girl_djvu.txt\n",
    "import re\n",
    "import nltk\n",
    "import math\n",
    "nltk.download('vader_lexicon')\n",
    "import spacy\n",
    "import random\n",
    "import inflect\n",
    "import datetime\n",
    "import community\n",
    "!pip install leidenalg\n",
    "import leidenalg\n",
    "import numpy as np\n",
    "import igraph as ig\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "from pprint import pprint\n",
    "from spacy import displacy\n",
    "from matplotlib import colors\n",
    "from operator import itemgetter\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "from matplotlib.pyplot import cm\n",
    "from itertools import permutations\n",
    "from scipy.ndimage.filters import gaussian_filter1d\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "\n",
    "#How to get pygraphviz\n",
    "#!wget https://anaconda.org/anaconda/pygraphviz/1.3/download/linux-64/pygraphviz-1.3-py36h14c3975_1.tar.bz2\n",
    "#!tar xvjf pygraphviz-1.3-py36h14c3975_1.tar.bz2\n",
    "#!cp -r lib/python3.6/site-packages/* /usr/local/lib/python3.6/dist-packages/\n",
    "\n",
    "#!sudo pip install python-igraph\n",
    "!sudo apt-get install graphviz libgraphviz-dev pkg-config\n",
    "!sudo apt-get install python-pip python-virtualenv\n",
    "!pip install pygraphviz\n",
    "\n",
    "import pygraphviz\n",
    "from networkx.drawing.nx_agraph import graphviz_layout\n",
    "\n",
    "!python -m spacy download en_core_web_lg\n",
    "import en_core_web_lg\n",
    "nlp = en_core_web_lg.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uZzyuVescCmD"
   },
   "outputs": [],
   "source": [
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aiQ2AAvTJhnU"
   },
   "outputs": [],
   "source": [
    "downloaded = drive.CreateFile({'id':\"1bKPRap5G-ZO2FLrS8WjdpScA6O4tgKAL\"}) \n",
    "downloaded.GetContentFile('TheDiaryOfAYoungGirl.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SUkKBDUzEhqz"
   },
   "source": [
    "## 1. Detecting Terms in Anne Frank's Diary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fluiMuCn-ID9"
   },
   "outputs": [],
   "source": [
    "#Regular expressions to look for when a new diary entry is beginning\n",
    "random.seed(1000)\n",
    "r = '(?:SUNDAY|MONDAY|TUESDAY|WEDNESDAY|THURSDAY|FRIDAY|SATURDAY), (?:JANUARY|FEBRUARY|MARCH|APRIL|MAY|JUNE|JULY|AUGUST|SEPTEMBER|OCTOBER|NOVEMBER|DECEMBER) \\d{1,2}, 19\\d{2}\\s*\\n'\n",
    "date_indexes = []\n",
    "#Open the file\n",
    "with open(\"TheDiaryOfAYoungGirl.txt\", encoding=\"utf8\", errors='ignore') as f:\n",
    "    lines = f.readlines()\n",
    "    for i, line in enumerate(lines):\n",
    "      # if '1943' in line:\n",
    "      #   #Make a new diary entry for date\n",
    "      if re.match(r, line):\n",
    "            date_indexes.append((line.strip(), i))\n",
    "    dia = {}\n",
    "    #Build dictionary of the date indices and their entries\n",
    "    for i in range(0, len(date_indexes) - 1):\n",
    "        cur_idx = date_indexes[i][1] + 1\n",
    "        next_idx = date_indexes[i + 1][1] - 1\n",
    "        dia.update({date_indexes[i][0]: ''.join(lines[cur_idx:next_idx])})\n",
    "#pprint(dia['FRIDAY, MARCH 17, 1944'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "A6VmDqcn_Pol"
   },
   "outputs": [],
   "source": [
    "dia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-5TNHE5G9eX5"
   },
   "outputs": [],
   "source": [
    "i = 0\n",
    "dd = []\n",
    "dto_list = []\n",
    "for key in dia.keys():\n",
    "    td = key.split(',')\n",
    "    td = td[1]+td[2]\n",
    "    td = td.lstrip()\n",
    "    dto = datetime.datetime.strptime(td, '%B %d %Y')\n",
    "    dto_list.append(dto)\n",
    "    if i == 0:\n",
    "        firstd = dto\n",
    "        dd.append(0)\n",
    "    else:\n",
    "        dd.append((dto-firstd).days)\n",
    "    i = i + 1\n",
    "#Consecutive day count from the first entry\n",
    "print(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F-ov9-Vcu8oe"
   },
   "outputs": [],
   "source": [
    "dic = {}\n",
    "i = 0\n",
    "#Build a dictionary of the days with their respective text\n",
    "for key, value in dia.items():\n",
    "    dic.update({dd[i] : value.replace('\\n','')})\n",
    "    i = i +1\n",
    "pprint(dic[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QuPS4XCNEy3t"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5u2INVZfu_T6"
   },
   "outputs": [],
   "source": [
    "p = inflect.engine()\n",
    "d_tags = {}\n",
    "\n",
    "#Remove these parts of existing entities, e.g. Dearest Kitty vs. Kitty\n",
    "def clean_up(X,ex_ls):\n",
    "    clean_l = ['Dearest','Dear','kittle','\\'s','\\\\','The','the','didn\\'t','hasn\\'t','doesn\\'t','every','8,1943','gize,'\n",
    "               ,'MAY 7,1944','wheeled','My','wasn\\'t','COMMENT','14g','I\\'m','couldn\\'t','Great','Old','future','looked'\n",
    "               ,'willi','de\\'','d\\'']\n",
    "    for clean in clean_l:\n",
    "        if (clean in X) and (X not in ex_ls):\n",
    "            X = X.replace(clean,'')\n",
    "            X = X.lstrip()\n",
    "            X = X.rstrip()\n",
    "    if p.singular_noun(X) and (X not in ex_ls):\n",
    "        X = p.singular_noun(X)\n",
    "    return X\n",
    "\n",
    "#Reg expressions to detect nlp entities and reclassify them to standardize\n",
    "def standardize(X,ex_ls):\n",
    "    #Things I'd like to standardize to aggregate better\n",
    "    rep = {'^anne':'Anne','^van d':'van D.','home guard':'Home Guard','^gies &':'Gies & Co.'\n",
    "           ,'secret annex':'Annex','^bep':'Bep','^miep':'Miep','^margot':'Margot'\n",
    "          ,'^peter':'Peter','hanneli':'Hanneli','hiller':'Hitler','^g.z. e. s.':'G.Z.',\n",
    "          '^olga meyen- dorff' : 'Olga Meyen-dorff','^ursul':'Ursula','kitty':'Kitty','das schon':'das schon'}\n",
    "    for key,value in rep.items():\n",
    "        if (re.match(key,X.replace('\"','').lower())) and (X not in ex_ls):\n",
    "            X = rep[key]\n",
    "    return X\n",
    "\n",
    "#Make entity tags of nlp for each diary or, later, sentence entry\n",
    "def make_entities(dic,keep_l):\n",
    "    for key, value in dic.items():\n",
    "        arr = []\n",
    "        doc = nlp(value.replace('\\n',''))\n",
    "        #Keep these types of nlp entities\n",
    "        #Typo/model error + german corrections\n",
    "        drop_t = ['it','you\\'re','he','his','her','ei ght','bd\\'\"dl','she','i\\'m','there','who','wasn\\'t','didn\\'t','that','here','we\\'ll'\n",
    "              ,'let','what','i','son','he\\'ll','re','wouldn\\'t','y\\'d','ridicu lou','lady','one','fed','i\\'d','\\'oh','you\\'d'\n",
    "              ,'s','shh','tranquthty','civthzed','now fifteen','hadn\\'t','sun','\\'i','she\\'ll','man','my hand','saw'\n",
    "              ,'\\'remem ber','re\\'re','you\\'ve','you\\'ll','shuffiing','he\\'d','bye','nitely','much tidier','semidarknes'\n",
    "              ,'you','te','mean-','conceited','my head','we\\'d','i\\'d lain','you\\'ve','mothproofed'\n",
    "              ,'\\'that','famthar','noth','mothjr','m r','clodiesline','diat','togedier','algebra','fadier','entsetzlich'\n",
    "              ,'nie zu ersetzen','nothina','\\'that','beina','future','secredy','hand','writina','dealina','enouah','runnina'\n",
    "              ,'unfortu','house','jun','\\'he','brrr','i. boy','mally','dread','pst','chin','phy','iii','head'\n",
    "              ,'cinema ater','sore throat','ketde','baker','seething','silent','tired','sister','lady','balli'\n",
    "              ,'ding-dong','sohn','ron','mor','himmelhoch jauchzend','zu tode','der mann','jan gy','das liebe','das schone'\n",
    "              ,'urn gotte','keg','eastern','du spritzt schon','quicksilver','espe','brea','miep, bep','gy','du bist doch eine'\n",
    "              ,'ich mach\\'','ruddy','kitty, bep','ifhe','pedes apostolorum','das schon','1','everYthing']\n",
    "\n",
    "        #Things inflect library handles poorly or to exclude from touching\n",
    "        ex_ls = ['Swiss','William Louis','Theseus','Zeus','Oedipus','Peleus','Orpheus','Hercules','Oasis','Maria Theresa','Myron'\n",
    "                ,'Phidias','Annex','Trees','Broks','Dutch Sasas','Kuperus']\n",
    "\n",
    "        for X in doc.ents:\n",
    "            s1 = X.text\n",
    "            if (X.label_ in keep_l):\n",
    "                s1 = clean_up(s1,ex_ls)\n",
    "                s1 = standardize(s1,ex_ls)\n",
    "                if (s1.lower() not in drop_t) and (s1):\n",
    "                    arr.append((s1, X.label_))\n",
    "        d_tags[key] = arr\n",
    "    return d_tags\n",
    "\n",
    "keep_l = ['PERSON','NORP','PRODUCT','ORG']\n",
    "d_tags = make_entities(dic,keep_l)\n",
    "pprint(d_tags[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "N_Zc_gPkRdWA"
   },
   "source": [
    "### 2. Dictionary of Attributes of Terms (NLP Entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tjmFTr6dxgJx"
   },
   "outputs": [],
   "source": [
    "chars = []\n",
    "full_tags = []\n",
    "tag_count_dic = {}\n",
    "j = 0\n",
    "for key, value in d_tags.items():\n",
    "    i = 0\n",
    "    tags = []\n",
    "    tag_count_dic[dto_list[j]] = {}\n",
    "    for elem in value:\n",
    "        full_tags.append(value[i])\n",
    "        chars.append(value[i][0])\n",
    "        tags.append(value[i][1])\n",
    "        i += 1\n",
    "    for tag in set(tags):\n",
    "        tag_count_dic[dto_list[j]][tag] = tags.count(tag)\n",
    "    j += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-sToDSYWR4TK"
   },
   "outputs": [],
   "source": [
    "tag_dic = {}\n",
    "node_dic = {}\n",
    "drop = ''\n",
    "for tag in full_tags:\n",
    "    if tag[0] not in tag_dic.keys():\n",
    "        tag_dic[tag[0]] = [tag[1]]\n",
    "    else:\n",
    "        tag_dic[tag[0]].append(tag[1])\n",
    "\n",
    "for key in tag_dic.keys():\n",
    "  node_dic[key] = {}\n",
    "  if key not in drop:\n",
    "    node_dic[key]['tag'] =  max(set(tag_dic[key]), key = tag_dic[key].count) \n",
    "\n",
    "node_dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BBS3evC3E_Fk"
   },
   "source": [
    "## 3. Time Series of Attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b8FwNMSyyOce"
   },
   "outputs": [],
   "source": [
    "dft = pd.DataFrame(tag_count_dic)\n",
    "dft = dft.fillna(0).transpose()\n",
    "plt.subplots(figsize=(18,12))\n",
    "columns = list(dft)\n",
    "i = 0\n",
    "cmap = cm.get_cmap('tab10')\n",
    "for column in columns:\n",
    "    y = gaussian_filter1d(dft[column],sigma=2.5)\n",
    "    x = list(tag_count_dic.keys())\n",
    "    plt.fill_between(x,y, color=cmap.colors[i], alpha=0.4)\n",
    "    plt.plot(x,y,color=cmap.colors[i],label=column)\n",
    "    i += 1\n",
    "\n",
    "plt.xlabel('Date',size=40)\n",
    "plt.ylabel('NLP Tag Count',size=40)\n",
    "plt.title('Tag Count Over Time',size=60)\n",
    "plt.ylim(ymin=0)\n",
    "plt.xlim(xmax=max(x),xmin=min(x))\n",
    "plt.legend(loc='upper right',prop={'size':14})\n",
    "plt.tick_params(axis='both', which='major', labelsize=20)\n",
    "plt.tick_params(axis='x', which='major',rotation=-45)\n",
    "#plt.setp(plt.xaxis.get_majorticklabels(), rotation=-45 ) \n",
    "#lines = dft.plot.line(figsize=(18,12))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ip1yYezFSO5O"
   },
   "source": [
    "### 4. Diary Co-Occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YteEHl82yQlU"
   },
   "outputs": [],
   "source": [
    "unique_chars = list(set(chars))\n",
    "unique_full_tags = list(set(full_tags))\n",
    "print(sorted(unique_chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S7aOkFU8yg9Y"
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns = unique_chars, index = unique_chars)\n",
    "df[:] = int(0)\n",
    "\n",
    "for key,value in d_tags.items():\n",
    "    arr = []\n",
    "    for elem in d_tags[key]:\n",
    "        arr.append(elem[0])\n",
    "    arr.append(elem[0])\n",
    "    for char1 in unique_chars:\n",
    "        for char2 in unique_chars:\n",
    "            if char1 in arr and char2 in arr:\n",
    "                df[char1][char2] += 1\n",
    "                df[char2][char1] += 1\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IbThnNotSufB"
   },
   "source": [
    "### 5. Sentential Co-Occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mfi8FVe4S1cW"
   },
   "outputs": [],
   "source": [
    "full_txt = ''\n",
    "sent_dic = {}\n",
    "sentences = []\n",
    "i = 0\n",
    "for z,v in dia.items():\n",
    "    v = v.replace('\\n','')\n",
    "    full_txt += v\n",
    "    dia[z] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oAfVj7Ls9m4G"
   },
   "outputs": [],
   "source": [
    "dia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "McAM6tVhS3N7"
   },
   "outputs": [],
   "source": [
    "def make_sentences(full_txt):\n",
    "    doc = nlp(full_txt)\n",
    "    sent_list = []\n",
    "    for i, token in enumerate(doc.sents):\n",
    "        for entity in unique_chars:\n",
    "            if entity in token.text:\n",
    "                sent_list.append(token.text)\n",
    "                break\n",
    "    return sent_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U1OUnGYcRghA"
   },
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "\n",
    "plt.figure(figsize=(12,6))\n",
    "\n",
    "filtered_txt = ''\n",
    "last_word = ''\n",
    "for word in full_txt.split():\n",
    "  if (word in unique_chars) and (word != last_word):\n",
    "    filtered_txt += \" \" + word\n",
    "    last_word = word\n",
    "\n",
    "# Generate a word cloud image\n",
    "wordcloud = WordCloud(background_color=\"black\",width=800,height=400).generate(filtered_txt)\n",
    "\n",
    "# Display the generated image:\n",
    "# the matplotlib way:\n",
    "plt.imshow(wordcloud, interpolation='bilinear')\n",
    "plt.axis(\"off\")\n",
    "plt.title('Wordcloud of Anne Frank\\'s Diary',size=40)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o1azyqF_KWCi"
   },
   "outputs": [],
   "source": [
    "i = 0\n",
    "nltk.download('vader_lexicon')\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "overall_sent_dic = {}\n",
    "for z,v in dia.items():\n",
    "  l1 = make_sentences(v)\n",
    "  for sentence in l1:\n",
    "    overall_sent_dic[i] = {}\n",
    "    overall_sent_dic[i]['sentence'] = sentence\n",
    "    overall_sent_dic[i]['date'] = z\n",
    "    i += 1\n",
    "overall_sent_dic[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YBlsTgT6TAdT"
   },
   "outputs": [],
   "source": [
    "sentiments = []\n",
    "sent_list = make_sentences(full_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "T_W6ZQzoTISP"
   },
   "outputs": [],
   "source": [
    "i = 0\n",
    "sent_dic = {}\n",
    "while i < len(sent_list):\n",
    "    sent_dic[i] = sent_list[i]\n",
    "    i += 1\n",
    "sent_tags = make_entities(sent_dic,keep_l)\n",
    "\n",
    "sent_tags[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DFHHxOw6103x"
   },
   "source": [
    "### 6. Sentiment of Sentential Co-Occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cmPxbsuvTYFo"
   },
   "outputs": [],
   "source": [
    "for key in sent_tags.keys():\n",
    "    sentiment = sid.polarity_scores(sent_list[key])\n",
    "    ent_list = []\n",
    "    for tag in sent_tags[key]:\n",
    "        ent_list.append(tag[0])\n",
    "    sentiments.append((ent_list,sentiment))\n",
    "sentiments[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z2Qq9ox_bvo3"
   },
   "outputs": [],
   "source": [
    "unique_chars.append('Mother')\n",
    "unique_chars.append('Father')\n",
    "unique_chars.append('Grandma')\n",
    "unique_chars.append('Mr. van Daan')\n",
    "unique_chars.append('Mr. van D.')\n",
    "unique_chars.append('Mrs. van Daan')\n",
    "unique_chars.append('Mrs. van D.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CEwys5TmODTN"
   },
   "outputs": [],
   "source": [
    "for z,v in overall_sent_dic.items():\n",
    "  unique_list = []\n",
    "  word_list = [i.lower().strip(',') for i in v['sentence'].split()]\n",
    "  word_list = [i.split('\\'')[0] for i in word_list]\n",
    "  for char in unique_chars:\n",
    "    for w in word_list:\n",
    "      if (char.lower()==w):\n",
    "        unique_list.append(char)\n",
    "    if (' ' in char) and (char in v['sentence']):\n",
    "      unique_list.append(char)\n",
    "  for word in unique_list:\n",
    "    for word2 in unique_list:\n",
    "      if ' ' in word2:\n",
    "        word3 = word2.split(' ')\n",
    "        for word4 in word3:\n",
    "          if (word == word4) and (word in unique_list):\n",
    "            unique_list.remove(word)\n",
    "  overall_sent_dic[z]['sentiment'] = sid.polarity_scores(v['sentence'])['compound']\n",
    "  overall_sent_dic[z]['cooccurrences'] = list(set(unique_list))\n",
    "overall_sent_dic[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-LQNFds5RxYW"
   },
   "outputs": [],
   "source": [
    "df_sent = pd.DataFrame.from_dict(overall_sent_dic).transpose()\n",
    "df_sent.index.name = 'SentID'\n",
    "df_sent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "B2CMgn7Q4-0s"
   },
   "outputs": [],
   "source": [
    "columns=['Date','Persons','Average Sentiment','Sentential Sentiment','Text']\n",
    "df_sent2 = pd.DataFrame(columns=columns)\n",
    "\n",
    "tup = []\n",
    "text_list = []\n",
    "dto,last_dto = 0,0\n",
    "for idx, (_, value) in enumerate(df_sent.iterrows()):\n",
    "    date = value.date\n",
    "    td = date.split(',')\n",
    "    td = td[1]+td[2]\n",
    "    td = td.lstrip()\n",
    "    dto = datetime.datetime.strptime(td, '%B %d %Y')\n",
    "    if ((dto != last_dto) or (idx==len(df_sent)-1)) and len(tup):\n",
    "      df_sent2.loc[last_idx] = pd.Series({'Date':last_dto, 'Persons':list(set(x for l in [i for (i,j) in tup] for x in l)), 'Average Sentiment':sum([i[1] for i in tup])/len([i[1] for i in tup]), 'Sentential Sentiment':tup,'Text':text_list})\n",
    "      text_list = []\n",
    "      tup = []\n",
    "    last_dto = dto\n",
    "    last_idx = idx\n",
    "    if len(value.cooccurrences) > 1:\n",
    "      tup.append((value.cooccurrences,value.sentiment))\n",
    "      text_list.append(value.sentence)\n",
    "df_sent2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dupS6tY6Tki0"
   },
   "outputs": [],
   "source": [
    "sentiments2 = []\n",
    "for element in sentiments:\n",
    "    ent_comb = list(permutations(element[0],2))\n",
    "    for ent_tup in ent_comb:\n",
    "      sentiments2.append((ent_tup,element[1]['compound']))\n",
    "sentiments2[0:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_F2VHh8CTwPH"
   },
   "outputs": [],
   "source": [
    "#Build dictionary of sentiments observed per edge\n",
    "sent_dic = {}\n",
    "for element in sentiments2:\n",
    "    if element[0] not in sent_dic:\n",
    "        sent_dic[element[0]] = [element[1]]\n",
    "    else:\n",
    "        sent_dic[element[0]].append(element[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BLVP-uKUIA9h"
   },
   "outputs": [],
   "source": [
    "duplicates = []\n",
    "for z in sent_dic:\n",
    "  if ((z[1],z[0]) in sent_dic) and ((z[1],z[0]) not in duplicates):\n",
    "    duplicates.append(z)\n",
    "\n",
    "for duplicate in duplicates:\n",
    "  del sent_dic[duplicate]\n",
    "\n",
    "len(sent_dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GjN6yDYrFdpE"
   },
   "source": [
    "## 7. Graph Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dGXck7FozmYJ"
   },
   "outputs": [],
   "source": [
    "def make_edges(df):\n",
    "    #Find edge values to determine max\n",
    "    edge_values = []\n",
    "    edge_list = {}\n",
    "    for index, row in df.iterrows():\n",
    "        i = 0\n",
    "        for col in row:\n",
    "            if col > 0 and (index != row.index[i]):\n",
    "                edge_values.append(col)\n",
    "            i += 1\n",
    "    \n",
    "    #Get max edge weight as percentage of that max edge value\n",
    "    for index, row in df.iterrows():\n",
    "        i = 0\n",
    "        for col in row:\n",
    "            #Remove edges with no weight, self-referential edges, and edges already added with inverse keys\n",
    "            if col > 0 and (index != row.index[i]) and ((df.columns[i], index) not in list(edge_list.keys())):\n",
    "                weight = float(col)/max(edge_values)\n",
    "                edge_list[(index, df.columns[i])] = weight\n",
    "            i += 1\n",
    "    return edge_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QOTUSaTIM48O"
   },
   "outputs": [],
   "source": [
    "def drop_nodes(df,drop_list):\n",
    "  df = df.drop(columns=drop_list)\n",
    "  df = df.drop(drop_list)\n",
    "  return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mHZQesAlKOW1"
   },
   "outputs": [],
   "source": [
    "def make_G(df,cooccurrence,filter,node_dic,sent_dic,sentiments2,ego='',drop=''):\n",
    "\n",
    "  if drop:\n",
    "    df2 = drop_nodes(df,[drop])\n",
    "  else:\n",
    "    df2 = df.copy()\n",
    "\n",
    "  df3 = df2.copy()\n",
    "  if filter:\n",
    "    df3[df3 < filter] = 0\n",
    "\n",
    "  edge_list = make_edges(df3)\n",
    "\n",
    "  if cooccurrence == 'sentences':\n",
    "    remove_edges = []\n",
    "    for edge in edge_list:\n",
    "      if (edge not in sent_dic) and ((edge[1],edge[0]) not in sent_dic):\n",
    "        remove_edges.append(edge)\n",
    "    for edge in remove_edges:\n",
    "      del edge_list[edge]\n",
    "\n",
    "  node_list = set([i for t in edge_list for i in t])\n",
    "\n",
    "  G = nx.Graph()\n",
    "  G.add_nodes_from(node_list)\n",
    "  G.add_edges_from(edge_list)\n",
    "\n",
    "  if ego:\n",
    "    G = nx.ego_graph(G, ego)\n",
    "\n",
    "  n_dic = {}\n",
    "  e_dic = {}\n",
    "  if cooccurrence=='diary':\n",
    "    for node in G.nodes():\n",
    "      n_dic[node] = {}\n",
    "      n_dic[node]['size'] = df3[node].sum()/2\n",
    "      n_dic[node]['tag'] = node_dic[node]['tag']\n",
    "    for edge in G.edges():\n",
    "      if edge in edge_list:\n",
    "        e_dic[edge] = {}\n",
    "        e_dic[edge]['width'] = edge_list[edge]\n",
    "      else:\n",
    "        e_dic[edge] = {}\n",
    "        e_dic[edge]['width'] = edge_list[(edge[1],edge[0])]\n",
    "\n",
    "  else:\n",
    "    for node in G.nodes():\n",
    "      count = 0\n",
    "      n_dic[node] = {}\n",
    "      for sent in sent_dic:\n",
    "        if (node == sent[0]) or (node == sent[1]):\n",
    "          count += 1\n",
    "      n_dic[node]['size'] = count\n",
    "      n_dic[node]['tag'] = node_dic[node]['tag']\n",
    "    for edge in G.edges():\n",
    "      if edge in sent_dic:\n",
    "        e_dic[edge] = {}\n",
    "        e_dic[edge]['sentiment'] = round(np.mean(sent_dic[edge]),2)\n",
    "        e_dic[edge]['width'] = [i[0] for i in sentiments2].count(edge)\n",
    "      elif (edge[1],edge[0]) in sent_dic:\n",
    "        e_dic[edge] = {}\n",
    "        e_dic[edge]['sentiment'] = round(np.mean(sent_dic[(edge[1],edge[0])]),2)\n",
    "        e_dic[edge]['width'] = [i[0] for i in sentiments2].count((edge[1],edge[0]))\n",
    "\n",
    "  return G,n_dic,e_dic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VOG6MxzT24JS"
   },
   "source": [
    "### 7a. Full Graph, Diary Co-Occurrent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zngSt1YwbFwU"
   },
   "outputs": [],
   "source": [
    "plt.subplots(figsize=(20,20))\n",
    "\n",
    "node_scalar = 0.7\n",
    "edge_scalar = 100\n",
    "\n",
    "G_everything,n_dic,e_dic = make_G(df,'diary',0,node_dic,sent_dic,sentiments2)\n",
    "\n",
    "pos = graphviz_layout(G_everything)\n",
    "\n",
    "nx.draw_networkx_edges(G_everything, pos, width=[e_dic[i]['width']*edge_scalar for i in G_everything.edges], edge_color='b', style='solid', alpha=0.05)\n",
    "nx.draw_networkx_nodes(G_everything, pos, with_labels=False,node_list=n_dic, font_size = 18, font_weight = 'bold',node_size=[n_dic[i]['size']*node_scalar for i in G_everything.nodes])\n",
    "nx.draw_networkx_labels(G_everything, pos, with_labels=True,node_list=n_dic, font_size = 12, font_weight = 'bold')\n",
    "\n",
    "plt.title('Diary Co-Occurrence of All Terms',size=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L9v5KV-L3bum"
   },
   "source": [
    "## 7b. Frequent Diary Co-Occurrent Terms (Anne Removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3oUc4wIW3VEb"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "plt.subplots(figsize=(10,10))\n",
    "\n",
    "node_scalar = 3\n",
    "edge_scalar = 10\n",
    "\n",
    "G,n_dic,e_dic = make_G(df,'diary',14,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "pos = graphviz_layout(G)\n",
    "\n",
    "nx.draw_networkx_edges(G, pos, width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges], edge_color='b', style='solid', alpha=0.8)\n",
    "nx.draw_networkx_nodes(G, pos, node_list=n_dic, font_size = 12, font_weight = 'bold',node_size=[n_dic[i]['size']*node_scalar for i in G.nodes])\n",
    "nx.draw_networkx_labels(G, pos, with_labels=True,node_list=n_dic, font_size = 12, font_weight = 'bold')\n",
    "\n",
    "plt.title('Diary Co-Occurrence of Filtered Terms (Anne Removed)',size=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iLH5flsn55Wj"
   },
   "source": [
    "## 7c. Sentiment of Sentential Co-Occurrent Terms (Anne Removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_G9E7IGX49-4"
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "\n",
    "plt.subplots(figsize=(10,10))\n",
    "\n",
    "node_scalar = 80\n",
    "edge_scalar = 5\n",
    "\n",
    "G,n_dic,e_dic = make_G(df,'sentences',8,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "pos = graphviz_layout(G,'neato')\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "\n",
    "if min(sentiments) >= 0:\n",
    "  min_sent = -1\n",
    "else:\n",
    "  min_sent = min(sentiments)\n",
    "\n",
    "if max(sentiments) <= 0:\n",
    "  max_sent = 1\n",
    "else:\n",
    "  max_sent = max(sentiments)\n",
    "from matplotlib.pyplot import cm\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent, vmax=max_sent)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm) #, orientation='horizontal')\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "nx.draw(G, pos, with_labels=True,node_list=n_dic, font_size = 12, font_weight = 'bold',node_size=[n_dic[i]['size']*node_scalar for i in G.nodes],width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges],edge_color=sentiment_colors)\n",
    "plt.title('Sentential Co-Occurrences of Filtered Terms\\n(Anne Removed) w/ Sentiment (1944)',size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F5MMiibWUt12"
   },
   "outputs": [],
   "source": [
    "def avg_sent(G,e_dic):\n",
    "  avsen_dic = {}\n",
    "  for node in G.nodes:\n",
    "    avsen_dic[node] = {}\n",
    "    sen_lis = []\n",
    "    tot_width = []\n",
    "    for edge in e_dic.keys():\n",
    "      if (node in edge[0]) or (node in edge[1]):\n",
    "        sen_lis.append(e_dic[edge]['sentiment']*e_dic[edge]['width'])\n",
    "        tot_width.append(e_dic[edge]['width'])\n",
    "    avsen_dic[node]['sentiment'] = sum(sen_lis)/len(sen_lis)\n",
    "    avsen_dic[node]['weighted_sentiment'] = sum(sen_lis)/(len(sen_lis)*sum(tot_width))\n",
    "  return avsen_dic\n",
    "\n",
    "sen_dic1 = avg_sent(G,e_dic)\n",
    "\n",
    "def bar_sent(sen_dic1):\n",
    "  from matplotlib.pyplot import cm\n",
    "  sen_dic2 = {}\n",
    "  for node in sen_dic1.keys():\n",
    "    sen_dic2[node] = sen_dic1[node]['weighted_sentiment']\n",
    "  D = {k: v for k, v in sorted(sen_dic2.items(), key=lambda item: item[1])}\n",
    "\n",
    "  sentiments = D.values()\n",
    "\n",
    "  if min(sentiments) >= 0:\n",
    "    min_sent = -1\n",
    "  else:\n",
    "    min_sent = min(sentiments)\n",
    "\n",
    "  if max(sentiments) <= 0:\n",
    "    max_sent = 1\n",
    "  else:\n",
    "    max_sent = max(sentiments)\n",
    "\n",
    "  cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "  norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent, vmax=max_sent)\n",
    "\n",
    "  sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "  fig, ax = plt.subplots()\n",
    "\n",
    "  # Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "  ax.set_ylabel('Average Sentiment',size=18)\n",
    "  ax.set_title('Weighted Sentiment by Node (1944)',size=24)\n",
    "  ax.legend()\n",
    "\n",
    "  fig.tight_layout()\n",
    "\n",
    "  fig.set_figheight(10)\n",
    "  fig.set_figwidth(8)\n",
    "\n",
    "  #sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "  #sm.set_array([])\n",
    "  #cbar = plt.colorbar(sm) #, orientation='horizontal')\n",
    "  #cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "  plt.bar(range(len(D)), list(D.values()), align='center',color=sentiment_colors)\n",
    "  plt.xticks(range(len(D)), list(D.keys()))\n",
    "\n",
    "  plt.xticks(rotation=90)\n",
    "  plt.axhline(y=0, color='k', linestyle='-')\n",
    "\n",
    "  plt.show()\n",
    "bar_sent(sen_dic1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BUBwpfUm6HGm"
   },
   "outputs": [],
   "source": [
    "pd.DataFrame.from_dict(sen_dic1).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CN01XE3r6pxl"
   },
   "outputs": [],
   "source": [
    "test_b = pd.DataFrame.from_dict(overall_sent_dic).T\n",
    "test_b.head(30)\n",
    "\n",
    "# for d in set(test_b['date']):\n",
    "#   print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ijYkgMpe6_a0"
   },
   "outputs": [],
   "source": [
    "test_b = test_b[[\"cooccurrences\",\"date\",\"sentiment\",\"sentence\"]]\n",
    "test_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Gwqdeo8QZAbC"
   },
   "outputs": [],
   "source": [
    "from scipy.stats import linregress\n",
    "\n",
    "def sent_slope(e_dic,overall_sent_dic):\n",
    "  slope_dic = {}\n",
    "  for edge in e_dic.keys():\n",
    "    #slope_dic[edge] = {}\n",
    "    days = []\n",
    "    sen_data = []\n",
    "    for k in overall_sent_dic.keys():\n",
    "      if (edge[0] in overall_sent_dic[k]['cooccurrences']) and (edge[1] in overall_sent_dic[k]['cooccurrences']):\n",
    "        days.append(k)\n",
    "        sen_data.append(overall_sent_dic[k]['sentiment'])\n",
    "    if days and sen_data:\n",
    "      slope = round(linregress(days,sen_data).slope,4)\n",
    "    else:\n",
    "      slope = 0\n",
    "    \n",
    "    if math.isnan(slope):\n",
    "      slope = 0\n",
    "\n",
    "    slope_dic[edge] = slope*1000\n",
    "\n",
    "  return slope_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BwnsWu1GZEmW"
   },
   "outputs": [],
   "source": [
    "from matplotlib.pyplot import cm\n",
    "\n",
    "plt.subplots(figsize=(20,10))\n",
    "\n",
    "node_scalar = 90\n",
    "edge_scalar = 3\n",
    "\n",
    "pos = graphviz_layout(G,'dot')\n",
    "\n",
    "sent_slope_dic = sent_slope(e_dic,overall_sent_dic)\n",
    "\n",
    "sentiments = [sent_slope_dic[i] for i in G.edges]\n",
    "\n",
    "if min(sentiments) >= 0:\n",
    "  min_sent = -1\n",
    "else:\n",
    "  min_sent = min(sentiments)\n",
    "\n",
    "if max(sentiments) <= 0:\n",
    "  max_sent = 1\n",
    "else:\n",
    "  max_sent = max(sentiments)\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=-1, vmax=1)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=-1, vmax=1))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm) #, orientation='horizontal')\n",
    "cbar.ax.set_ylabel('Slope of Sentiment',size=30,rotation=270,labelpad=30)\n",
    "\n",
    "nx.draw(G, pos, with_labels=True,node_list=n_dic, font_size = 12, font_weight = 'bold',node_size=[n_dic[i]['size']*node_scalar for i in G.nodes],width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges],edge_color=sentiment_colors)\n",
    "plt.title('Sentential Co-Occurrences of Filtered Terms\\n(Anne Removed) w/ Sentiment Slope',size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lOspfggDGfQ4"
   },
   "outputs": [],
   "source": [
    "def avg_sent_slope(G,sent_slope_dic,e_dic):\n",
    "  avsen_dic = {}\n",
    "  for node in G.nodes:\n",
    "    avsen_dic[node] = {}\n",
    "    sen_lis = []\n",
    "    tot_width = []\n",
    "    for edge in G.edges:\n",
    "      if (node in edge[0]) or (node in edge[1]):\n",
    "        sen_lis.append(sent_slope_dic[edge]*e_dic[edge]['width'])\n",
    "        tot_width.append(e_dic[edge]['width'])\n",
    "    print(node,sen_lis)\n",
    "    avsen_dic[node]['sentiment'] = sum(sen_lis)/len(sen_lis)\n",
    "    avsen_dic[node]['weighted_sentiment'] = sum(sen_lis)/(len(sen_lis)*sum(tot_width))\n",
    "  return avsen_dic\n",
    "\n",
    "sen_dic1 = avg_sent_slope(G,sent_slope_dic,e_dic)\n",
    "\n",
    "def bar_sent(sen_dic1):\n",
    "  from matplotlib.pyplot import cm\n",
    "  sen_dic2 = {}\n",
    "  for node in sen_dic1.keys():\n",
    "    sen_dic2[node] = sen_dic1[node]['weighted_sentiment']\n",
    "  D = {k: v for k, v in sorted(sen_dic2.items(), key=lambda item: item[1])}\n",
    "\n",
    "  sentiments = D.values()\n",
    "\n",
    "  if min(sentiments) >= 0:\n",
    "    min_sent = -1\n",
    "  else:\n",
    "    min_sent = min(sentiments)\n",
    "\n",
    "  if max(sentiments) <= 0:\n",
    "    max_sent = 1\n",
    "  else:\n",
    "    max_sent = max(sentiments)\n",
    "\n",
    "  cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "  norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent, vmax=max_sent)\n",
    "\n",
    "  sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "  fig, ax = plt.subplots()\n",
    "\n",
    "  # Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "  ax.set_ylabel('Average Sentiment',size=18)\n",
    "  ax.set_title('Weighted Sentiment by Node (1944)',size=24)\n",
    "  ax.legend()\n",
    "\n",
    "  fig.tight_layout()\n",
    "\n",
    "  fig.set_figheight(10)\n",
    "  fig.set_figwidth(8)\n",
    "\n",
    "  #sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "  #sm.set_array([])\n",
    "  #cbar = plt.colorbar(sm) #, orientation='horizontal')\n",
    "  #cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "  plt.bar(range(len(D)), list(D.values()), align='center',color=sentiment_colors)\n",
    "  plt.xticks(range(len(D)), list(D.keys()))\n",
    "\n",
    "  plt.xticks(rotation=90)\n",
    "  plt.axhline(y=0, color='k', linestyle='-')\n",
    "\n",
    "  plt.show()\n",
    "bar_sent(sen_dic1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6hd-E05u7RmR"
   },
   "source": [
    "### 7d. Clustering Functions for Graph Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Djbs0EuO7Pjh"
   },
   "outputs": [],
   "source": [
    "nlp_misclassifieds = {'Miep':'PERSON','Moortje':'PERSON'}\n",
    "for node in nlp_misclassifieds:\n",
    "  if node in node_dic:\n",
    "    node_dic[node]['tag'] = nlp_misclassifieds[node]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mJK2TH1O6m0E"
   },
   "outputs": [],
   "source": [
    "def make_clusters_async(G,clusters=5):\n",
    "    from networkx.algorithms import community\n",
    "    from matplotlib import colors\n",
    "    import itertools\n",
    "    communities = list(nx.community.asyn_fluidc(G, clusters))\n",
    "    i = 0\n",
    "    node = {}\n",
    "    cmap = cm.get_cmap('Set3')\n",
    "    node_colors = []\n",
    "    for entity in sorted(G.nodes):\n",
    "        for n, com in enumerate(list(communities)):\n",
    "            if entity in com:\n",
    "                color = colors.to_hex(cmap.colors[n])\n",
    "        node_colors.append(color)\n",
    "    return node_colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8-Rhf-pE65Bm"
   },
   "outputs": [],
   "source": [
    "test_nodes = []\n",
    "tag_to_train = 'NORP'\n",
    "for node in G.nodes:\n",
    "  if node in node_dic.keys():\n",
    "    if node_dic[node]['tag'] == tag_to_train:\n",
    "      test_nodes.append(node)\n",
    "test_nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "C1gC3Vq6-oeV"
   },
   "outputs": [],
   "source": [
    "tags_that_exist = []\n",
    "for node in G.nodes:\n",
    "  tags_that_exist.append(n_dic[node]['tag'])\n",
    "tags_that_exist = set(tags_that_exist)\n",
    "tags_that_exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JftbJEtH-qnS"
   },
   "outputs": [],
   "source": [
    "i = 0\n",
    "z = 0\n",
    "h = 0\n",
    "clusters = len(tags_that_exist)\n",
    "while (i != (len(test_nodes)-h)) or (min(color_l) < (4-h)):\n",
    "  node_colors = make_clusters_async(G,clusters)\n",
    "  j = 0\n",
    "  color_dic = {}\n",
    "  while j < len(n_dic):\n",
    "      color_dic[list(n_dic)[j]] = node_colors[j]\n",
    "      j += 1\n",
    "  max_color = {}\n",
    "  for tnode in test_nodes:\n",
    "      count = 0\n",
    "      for tnode2 in test_nodes:\n",
    "          if tnode != tnode2:\n",
    "              if color_dic[tnode] == color_dic[tnode2]:\n",
    "                  count += 1\n",
    "          max_color[tnode] = count\n",
    "  i = max(max_color.values())\n",
    "  color_l = []\n",
    "  for color in set(node_colors):\n",
    "    color_l.append(node_colors.count(color))\n",
    "  if z == 1000:\n",
    "    h += 1\n",
    "    z = 0\n",
    "  z += 1\n",
    "\n",
    "  if h > 4:\n",
    "    break\n",
    "\n",
    "x = 0\n",
    "color_comm_membership = {}\n",
    "for value in set(color_dic.values()):\n",
    "  cluster = []\n",
    "  for node in n_dic:\n",
    "    if color_dic[node] == value:\n",
    "      cluster.append(node)\n",
    "  color_comm_membership[x] = cluster\n",
    "  x += 1\n",
    "i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b3AjEUE4_ObI"
   },
   "outputs": [],
   "source": [
    "cmap = cm.get_cmap('Set3')\n",
    "for i in color_comm_membership:\n",
    "  for node in color_comm_membership[i]:\n",
    "    n_dic[node]['fluid_color'] = cmap.colors[i]\n",
    "n_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zl_gzq55_qxw"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "def make_coordinates(num,coord_scale=9):\n",
    "  angle = 2*math.pi/num#+random.uniform(0, 1)\n",
    "  angle_list = []\n",
    "  i = 0\n",
    "  while i < num:\n",
    "    angle_list.append(i*angle)\n",
    "    i += 1\n",
    "  coordinates = []\n",
    "  coordinates.append((0,-1*coord_scale))\n",
    "  j = 1\n",
    "  while j < len(angle_list):\n",
    "    coordinates.append((round(math.cos(angle_list[j]-math.pi/2)*coord_scale,2),round(math.sin(angle_list[j]-math.pi/2)*coord_scale,2)))\n",
    "    j += 1\n",
    "  return coordinates\n",
    "make_coordinates(len(tags_that_exist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ez1IsA7w_1--"
   },
   "outputs": [],
   "source": [
    "def sort_position_list(pos,x_dic,x_list,nodes=n_dic):\n",
    "  pos_list = []\n",
    "  num_nodes = []\n",
    "  i = 0\n",
    "  for l in x_list:\n",
    "    position_dic = {}\n",
    "    for node in nodes:\n",
    "      if x_dic[node]['tag'] == l:\n",
    "        position_dic[node] = pos[node]\n",
    "    if position_dic:\n",
    "      pos_list.append(position_dic)\n",
    "\n",
    "  def getNumKeys(dict1):\n",
    "    return len(dict1.keys())\n",
    "  pos_list=sorted(pos_list, key=getNumKeys)[::-1]\n",
    "  return pos_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9MypOKwfAC3z"
   },
   "outputs": [],
   "source": [
    "pos_list = sort_position_list(pos,n_dic,tags_that_exist)\n",
    "coordinates = make_coordinates(len(tags_that_exist))\n",
    "def sort_coord(coordinates,pos_list):\n",
    "  i=0\n",
    "  y_list = []\n",
    "  miny,maxy = (), ()\n",
    "  coord_list = []\n",
    "  tag_list = []\n",
    "  tag_type = ''\n",
    "  for coord in coordinates:\n",
    "    tag_type = n_dic[list(pos_list[i].keys())[0]]['tag']\n",
    "    y_list.append(coord[1])\n",
    "    tag_list.append((tag_type,len(pos_list[i].keys())))\n",
    "    i += 1\n",
    "\n",
    "  def getKey(item):\n",
    "    return item[1]\n",
    "  tag_list=sorted(tag_list, key=getKey)[::-1]\n",
    "\n",
    "  for coord in coordinates:\n",
    "    if coord[1] == max(y_list):\n",
    "      maxy = coord\n",
    "    if coord[1] == min(y_list):\n",
    "      miny = coord\n",
    "  coord_list.append(miny)\n",
    "  coord_list.append(maxy)\n",
    "  for coord in coordinates:\n",
    "    if (coord != miny) and (coord != maxy):\n",
    "      coord_list.append(coord)\n",
    "  return coord_list,tag_list\n",
    "\n",
    "coord_list,tag_list = sort_coord(coordinates,pos_list)\n",
    "coord_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Dw2j-74nAaxz"
   },
   "outputs": [],
   "source": [
    "interim_tag = ['']*len(tags_that_exist)\n",
    "def getIndexTuple(tup_list,tag):\n",
    "  i = 0\n",
    "  for tup in tup_list:\n",
    "    index = i\n",
    "    if tup[0] == tag:\n",
    "      break\n",
    "    i += 1\n",
    "  return index\n",
    "    \n",
    "for tag in tags_that_exist:\n",
    "  ind = getIndexTuple(tag_list,tag)\n",
    "  interim_tag[ind]=tag\n",
    "\n",
    "tags_that_exist = interim_tag\n",
    "tags_that_exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "foOYcB8oAkCM"
   },
   "outputs": [],
   "source": [
    "def circular_position(x_list,coord_list,pos,pos_list,x_dic):\n",
    "  cir_pos_list = []\n",
    "  #k = 0.8\n",
    "  i = 0\n",
    "  radius_scalar = 0.25 #2.5/coord_scale\n",
    "  for l in x_list:\n",
    "    G1 = nx.Graph()\n",
    "    node_list = []\n",
    "    for node in G.nodes:\n",
    "      if x_dic[node]['tag'] == l:\n",
    "        node_list.append(node)\n",
    "    G1.add_nodes_from(node_list)\n",
    "    #pos = nx.spring_layout(G1, k, iterations=17)\n",
    "    pos = nx.circular_layout(G1,scale=len(G1.nodes)*radius_scalar)\n",
    "    #print(pos)\n",
    "    for key in pos.keys():\n",
    "      pos[key] = [pos[key][0]+coord_list[i][0],pos[key][1]+coord_list[i][1]]\n",
    "    cir_pos_list.append(pos)\n",
    "    i += 1\n",
    "  return cir_pos_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ooChiN7IAurC"
   },
   "outputs": [],
   "source": [
    "pos2 = {}\n",
    "cir_pos_list = circular_position(tags_that_exist,coord_list,pos,pos_list,n_dic)\n",
    "i=0\n",
    "while i < len(cir_pos_list):\n",
    "  for key in cir_pos_list[i].keys():\n",
    "    pos2[key] = cir_pos_list[i][key]\n",
    "  i += 1\n",
    "pos2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ssSFhD7WBYxE"
   },
   "outputs": [],
   "source": [
    "!pip install circle-fit\n",
    "import circle_fit as cf\n",
    "\n",
    "def show_circle_layout(cir_pos_list,min_radius=1):\n",
    "  cmap = cm.get_cmap('Set3')\n",
    "  circle_list = []\n",
    "  xrange = []\n",
    "  yrange = []\n",
    "  rmax = 0\n",
    "  fig, ax = plt.subplots(figsize=(10,10))\n",
    "  ax.set_aspect(\"equal\")\n",
    "  i = 0\n",
    "  for element in cir_pos_list:\n",
    "    points = list(element.values())\n",
    "    if len(points) > 1:\n",
    "      xc,yc,r,_ = cf.least_squares_circle(points)\n",
    "    else:\n",
    "      xc,yc,r = pos2[list(element.keys())[0]][0],pos2[list(element.keys())[0]][1],min_radius\n",
    "    #circle = plt.Circle((xc, yc), r, color='r')\n",
    "    #fig, ax = plt.subplots() # note we must use plt.subplots, not plt.subplt\n",
    "    # (or if you have an existing figure)\n",
    "    #fig = plt.gcf()\n",
    "    #ax = fig.gca()\n",
    "    if r > rmax:\n",
    "      rmax = r\n",
    "    circle_list.append((xc,yc,r))\n",
    "    xrange.append(xc-r)\n",
    "    xrange.append(xc+r)\n",
    "    yrange.append(yc-r)\n",
    "    yrange.append(yc+r)\n",
    "    c=plt.Circle((xc,yc),r,color=cmap.colors[i])\n",
    "    ax.add_artist(c)\n",
    "      # Show/save figure as desired.\n",
    "  plt.xlim(min(xrange),max(xrange))\n",
    "  plt.ylim(min(yrange),max(yrange))\n",
    "  plt.show()\n",
    "  return circle_list,rmax\n",
    "\n",
    "  #fig.savefig('plotcircles.png')\n",
    "circle_list,rmax = show_circle_layout(cir_pos_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tZku3-UpBhI5"
   },
   "outputs": [],
   "source": [
    "tag_cir_dic = {}\n",
    "i = 0\n",
    "for tag in tags_that_exist:\n",
    "  tag_cir_dic[tag] = circle_list[i]\n",
    "  i += 1\n",
    "tag_cir_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aguE3yo6Oxbk"
   },
   "outputs": [],
   "source": [
    "circle_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LFESETQUBqBn"
   },
   "outputs": [],
   "source": [
    "from sympy.solvers import solve\n",
    "from sympy import Symbol\n",
    "def find_xydelta(circle_list,rmax,offset=2):\n",
    "  i = 0\n",
    "  ref_circle = ()\n",
    "  xychange = []\n",
    "  for circle in circle_list:\n",
    "    r = circle[2]\n",
    "    if (r == rmax) and (i == 0):\n",
    "      ref_circle = circle\n",
    "      i = 1\n",
    "  i = 0\n",
    "  for circle in circle_list:\n",
    "    x_cir,y_cir,r = circle[0],circle[1],circle[2]\n",
    "    x_ref,y_ref,r_ref = ref_circle[0],ref_circle[1],ref_circle[2]\n",
    "    x_move = 0\n",
    "    y_move = 0\n",
    "    if (r == rmax) and (i == 0):\n",
    "      i = 1\n",
    "    #elif abs(round(y_cir/y_ref,3)) == 1:\n",
    "    #  y_move = y_cir-y_ref + offset + r_ref\n",
    "    elif round(x_cir/x_ref,3) == 1:\n",
    "      if y_cir > y_ref:\n",
    "        y_move = y_cir-y_ref-r_ref-offset\n",
    "      elif y_ref > y_cir:\n",
    "        y_move = -(y_ref-y_cir-r_ref-offset)\n",
    "    elif round(y_cir/y_ref,3) == 1:\n",
    "      if x_cir > x_ref:\n",
    "        x_move = x_cir-x_ref-r_ref-offset\n",
    "      elif x_ref > x_cir:\n",
    "        x_move = -(x_ref-x_cir-r_ref-offset)\n",
    "    else:\n",
    "      m = (y_cir-y_ref)/(x_cir-x_ref)\n",
    "      b = y_cir-m*x_cir\n",
    "      x = Symbol('x')\n",
    "      ref_answer = solve((x - x_ref)**2+(m*x+b-y_ref)**2-(r_ref+offset)**2, x)\n",
    "      x_ref_ans = 0\n",
    "      for a in ref_answer:\n",
    "        if (x_ref < a < x_cir) or (x_ref > a > x_cir):\n",
    "          x_ref_ans = a\n",
    "        y_ref_ans = x_ref_ans*m+b\n",
    "      answer = solve((x - x_cir)**2+(m*x+b-y_cir)**2-(r)**2, x)\n",
    "      x_ans = 0\n",
    "      for a in answer:\n",
    "        if (x_ref < a < x_cir) or (x_ref > a > x_cir):\n",
    "          x_ans = a\n",
    "        y_ans = x_ans*m+b\n",
    "      x_move = round(x_ans - x_ref_ans,2)\n",
    "      y_move = round(y_ans - y_ref_ans,2)\n",
    "    xychange.append((-x_move,-y_move))\n",
    "  return xychange\n",
    "\n",
    "xychange = find_xydelta(circle_list,rmax)\n",
    "xychange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6FkaUDZaB3Wz"
   },
   "outputs": [],
   "source": [
    "def centralize_circles(cir_pos_list,xychange):\n",
    "  i = 0\n",
    "  pos_list2 = []\n",
    "  while i < len(cir_pos_list):\n",
    "    for key in cir_pos_list[i].keys():\n",
    "      new_node_list2 = {}\n",
    "      new_node_list2[key] = [cir_pos_list[i][key][0]+xychange[i][0],cir_pos_list[i][key][1]+xychange[i][1]]\n",
    "      pos_list2.append(new_node_list2)\n",
    "    i += 1\n",
    "  pos_list2\n",
    "\n",
    "  final_pos_dic = {}\n",
    "  i=0\n",
    "  while i < len(pos_list2):\n",
    "    for key in pos_list2[i].keys():\n",
    "      final_pos_dic[key] = pos_list2[i][key]\n",
    "    i += 1\n",
    "  return final_pos_dic\n",
    "\n",
    "final_pos_dic = centralize_circles(cir_pos_list,xychange)\n",
    "final_pos_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qV31Jxf2CAy4"
   },
   "outputs": [],
   "source": [
    "def add_center_circles(xychange,circle_list,names):\n",
    "  i = 0\n",
    "  G2 = nx.Graph()\n",
    "  pos_dic = {}\n",
    "  center_dic = {}\n",
    "  sizes = []\n",
    "  r_dic = {}\n",
    "  node_colors = []\n",
    "  cmap = cm.get_cmap('Set3')\n",
    "\n",
    "  while i < len(circle_list):\n",
    "    name = names[i]\n",
    "    x = circle_list[i][0]+xychange[i][0]\n",
    "    y = circle_list[i][1]+xychange[i][1]\n",
    "    r_dic[name] = circle_list[i][2]*100000\n",
    "    pos_dic[name] = [x,y]\n",
    "    center_dic[name] = circle_list[i][2]\n",
    "    node_colors.append(cmap.colors[i])\n",
    "    i += 1\n",
    "  \n",
    "  G2.add_nodes_from(center_dic)\n",
    "  for node in G2.nodes:\n",
    "    sizes.append(r_dic[node])\n",
    "\n",
    "  return G2,pos_dic,sizes,node_colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8NwBZJprCJzb"
   },
   "outputs": [],
   "source": [
    "def overall_graph(node_dic,edge_dic,tag_cir_dic,node_scalar=10000,edge_scalar=20):\n",
    "  cmap = cm.get_cmap('Set3')\n",
    "  tag_node_dic = {}\n",
    "  tag_edge_dic = {}\n",
    "  tag_pos_dic = {}\n",
    "  i = 0\n",
    "  for tag in set(tag_cir_dic):\n",
    "    node_num = 0\n",
    "    tag_node_dic[tag] = {}\n",
    "    size_l = []\n",
    "    for node in node_dic:\n",
    "      if node_dic[node]['tag'] == tag:\n",
    "        node_num += 1\n",
    "        size_l.append(node_dic[node]['size'])\n",
    "    tag_node_dic[tag]['node_num'] = node_num\n",
    "    tag_node_dic[tag]['color'] = cmap.colors[i]\n",
    "    tag_pos_dic[tag] = (tag_cir_dic[tag][0],tag_cir_dic[tag][1])\n",
    "    tag_node_dic[tag]['radius'] = tag_cir_dic[tag][2]\n",
    "    tag_node_dic[tag]['avg_size'] = sum(size_l)/len(size_l)\n",
    "    i += 1\n",
    "\n",
    "  for tag in tag_cir_dic:\n",
    "    for tag2 in tag_cir_dic:\n",
    "      if (tag != tag2) and ((tag2,tag) not in tag_edge_dic.keys()):\n",
    "        sentiment_l = []\n",
    "        width_l = []\n",
    "        for edge in edge_dic:\n",
    "          if (((node_dic[edge[0]]['tag'] == tag) and (node_dic[edge[1]]['tag'] ==  tag2)) or ((node_dic[edge[1]]['tag'] == tag) and (node_dic[edge[0]]['tag'] ==  tag2))):\n",
    "            sentiment_l.append(edge_dic[edge]['sentiment'])\n",
    "            width_l.append(edge_dic[edge]['width'])\n",
    "        if sentiment_l and width_l:\n",
    "          tag_edge_dic[(tag,tag2)] = {}\n",
    "          tag_edge_dic[(tag,tag2)]['width'] = round(sum(width_l) / len(width_l),2)\n",
    "          tag_edge_dic[(tag,tag2)]['sentiment'] = round(sum(sentiment_l) / len(sentiment_l),2)\n",
    "  #tag_edge_dic = {z: v for z, v in tag_edge_dic.items() if v}\n",
    "  G_all = nx.Graph()\n",
    "  G_all.add_nodes_from(tag_node_dic)\n",
    "  G_all.add_edges_from(tag_edge_dic)\n",
    "\n",
    "  for edge in G_all.edges:\n",
    "    if edge not in tag_edge_dic:\n",
    "      tag_edge_dic[edge] = {}\n",
    "      tag_edge_dic[edge]['width'] = tag_edge_dic[(edge[1],edge[0])]['width']\n",
    "      tag_edge_dic[edge]['sentiment'] = tag_edge_dic[(edge[1],edge[0])]['sentiment']\n",
    "      del tag_edge_dic[(edge[1],edge[0])]\n",
    "\n",
    "  return G_all,tag_node_dic,tag_edge_dic,tag_pos_dic\n",
    "G_all,tag_node_dic,tag_edge_dic,tag_pos_dic = overall_graph(n_dic,e_dic,tag_cir_dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VMqCmmX9Uvad"
   },
   "source": [
    "## 7e. Overall Clustering of Sentential Co-Occurences by Term Attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ye1KA2QMparS"
   },
   "outputs": [],
   "source": [
    "def get_att_assort(G,n_dic,att='tag'):\n",
    "  Gf=nx.Graph()\n",
    "  i = 0\n",
    "  att_list = []\n",
    "  for node in G.nodes:\n",
    "    att_list.append(n_dic[node][att])\n",
    "  att_list =  set(att_list)\n",
    "  for x in att_list:\n",
    "    node_list = []\n",
    "    for node in G.nodes:\n",
    "      if x == n_dic[node][att]:\n",
    "        node_list.append(node)\n",
    "    Gf.add_nodes_from(node_list,att=n_dic[node_list[0]][att])\n",
    "    i += 1\n",
    "  Gf.add_edges_from(G.edges)\n",
    "  return round(nx.attribute_assortativity_coefficient(Gf,'att'),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-01oXnkoECRa"
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "plt.figure(figsize=(15,15))\n",
    "\n",
    "edge_scalar = 20\n",
    "node_scalar = 30000\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in e_dic]\n",
    "node_size = [tag_node_dic[i]['radius']*node_scalar for i in G_all.nodes]\n",
    "colors = [tag_node_dic[i]['color'] for i in G_all.nodes]\n",
    "\n",
    "tag_sentiment = [tag_edge_dic[i]['sentiment'] for i in G_all.edges]\n",
    "widths = [math.log(tag_edge_dic[i]['width']+1)*edge_scalar for i in G_all.edges]\n",
    "\n",
    "cmap = plt.cm.viridis_r\n",
    "cmaplist = [cmap(i) for i in range(cmap.N)]\n",
    "cmaplist[0] = (1.0,1.0,1.0,1.0)\n",
    "cmap = matplotlib.colors.LinearSegmentedColormap.from_list('mcm',cmaplist, cmap.N)\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent, vmax=max_sent)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in tag_sentiment]\n",
    "\n",
    "nx.draw(G_all,tag_pos_dic, with_labels=True, font_size = 18, font_weight = 'bold',alpha=1,node_size=node_size,node_color=colors)\n",
    "\n",
    "nx.draw_networkx_edges(G_all,tag_pos_dic,edgelist=G_all.edges,width=widths,edge_color=sentiment_colors, alpha=1)\n",
    "\n",
    "plt.title('Sentential Co-Occurrence by\\nNLP Attribute Summarization',size=40)\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm) #, orientation='horizontal')\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "plt.xlim(-12.5,12.5)\n",
    "plt.ylim(-15.2,13.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GBTUFqEiYrTd"
   },
   "source": [
    "## 7f. Sentential Co-Occurence Clustering by Term Attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DLhrQ9tiKbV3"
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "from mpl_toolkits.axes_grid1 import AxesGrid\n",
    "\n",
    "plt.figure(figsize=(20,20))\n",
    "\n",
    "edge_scalar = 4\n",
    "node_scalar = 200\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "node_size = [n_dic[i]['size']*node_scalar for i in G.nodes]\n",
    "colors = [n_dic[i]['fluid_color'] for i in G.nodes]\n",
    "\n",
    "sentiment = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "widths = [math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges]\n",
    "\n",
    "cmap = plt.cm.viridis_r\n",
    "cmaplist = [cmap(i) for i in range(cmap.N)]\n",
    "cmaplist[0] = (1.0,1.0,1.0,1.0)\n",
    "cmap = matplotlib.colors.LinearSegmentedColormap.from_list('mcm',cmaplist, cmap.N)\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiment]\n",
    "\n",
    "nx.draw(G, final_pos_dic, with_labels=True, font_size = 18, font_weight = 'bold',node_size = [n_dic[i]['size']*node_scalar for i in n_dic], node_color = colors, width = widths)\n",
    "\n",
    "nx.draw_networkx_edges(G, final_pos_dic,edgelist=G.edges,width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges], alpha=1, edge_color=sentiment_colors)\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "plt.title('Sentential Co-Occurrences by\\nFluid Community (Color), Assort = {}\\n NLP Object (Cluster), Assort = {}'.format(get_att_assort(G,n_dic,att='fluid_color'),get_att_assort(G,n_dic)),size=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SxMKFpl6PSvM"
   },
   "outputs": [],
   "source": [
    "import community\n",
    "import operator\n",
    "\n",
    "partition_lo = community.best_partition(G)\n",
    "\n",
    "kk=len(set(partition_lo.values()))\n",
    "modularity_lo=community.modularity(partition_lo, G, weight='weight')\n",
    "print('The graph has', kk, 'Louvain communities and modularity coefficient equal to', modularity_lo)\n",
    "\n",
    "cml=[]\n",
    "for j in range(kk):\n",
    "    cj=[]\n",
    "    for k,v in partition_lo.items():\n",
    "        if v==j:\n",
    "            cj.append(k)\n",
    "    cml.append(cj)\n",
    "cmd={}\n",
    "for j in range(kk):\n",
    "    cmd[tuple(cml[j])]=len(cml[j])\n",
    "cmd\n",
    "ocmd=sorted(cmd.items(), key=operator.itemgetter(1), reverse=True)\n",
    "eocmd=enumerate(ocmd)\n",
    "commsd_lo={}\n",
    "louvain_comm_membership={}\n",
    "for i in eocmd:\n",
    "    print('Community', i[0], 'with', i[1][1], 'nodes:', list(i[1][0]))\n",
    "    commsd_lo[i[0]]=i[1][1] \n",
    "    ts=list(i[1][0])\n",
    "    louvain_comm_membership[i[0]]=ts "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IJ7lwXNvfFNC"
   },
   "outputs": [],
   "source": [
    "g = ig.Graph(directed=False)\n",
    "g.add_vertices(list(G.nodes))\n",
    "g.add_edges(G.edges)\n",
    "\n",
    "partition_le = leidenalg.find_partition(g, leidenalg.ModularityVertexPartition);\n",
    "\n",
    "partition_nodes = []\n",
    "for part_list in partition_le:\n",
    "  part = [list(G.nodes())[i] for i in part_list]\n",
    "  partition_nodes.append(part)\n",
    "\n",
    "cml=list(partition_nodes)\n",
    "leiden_partition={}\n",
    "for i,j in list(enumerate(cml)):\n",
    "  for jj in j:\n",
    "    leiden_partition[jj]=i\n",
    "modularity_le=community.modularity(leiden_partition, G, weight='weight')\n",
    "print('The graph has', kk, 'Leiden communities and modularity coefficient equal to', modularity_le)\n",
    "\n",
    "cmd={}\n",
    "kk=len(cml)\n",
    "for j in range(kk):\n",
    "  cmd[tuple(cml[j])]=len(cml[j])\n",
    "cmd\n",
    "\n",
    "ocmd=sorted(cmd.items(), key=operator.itemgetter(1), reverse=True)\n",
    "eocmd=enumerate(ocmd)\n",
    "commsd_le={}\n",
    "leiden_comm_membership={}\n",
    "for i in eocmd:\n",
    "    print('Community', i[0], 'with', i[1][1], 'nodes:', list(i[1][0]))\n",
    "    commsd_le[i[0]]=i[1][1] \n",
    "    ts=list(i[1][0])\n",
    "    leiden_comm_membership[i[0]]=ts "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3-yslxrJsYYc"
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "from mpl_toolkits.axes_grid1 import AxesGrid\n",
    "\n",
    "for z,v in louvain_comm_membership.items():\n",
    "  for node in v:\n",
    "    n_dic[node]['louvain_partition'] = 'Louvain_Comm_' + str(z)\n",
    "\n",
    "plt.figure(figsize=(20,20))\n",
    "\n",
    "cmap = cm.get_cmap('Set3')\n",
    "\n",
    "edge_scalar = 3\n",
    "node_scalar = 300\n",
    "\n",
    "communities = list(set([n_dic[i]['louvain_partition'] for i in G.nodes]))\n",
    "sentiments = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "node_size = [n_dic[i]['size']*node_scalar for i in G.nodes]\n",
    "colors = [cmap.colors[communities.index(n_dic[i]['louvain_partition'])] for i in G.nodes]\n",
    "\n",
    "sentiment = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "widths = [math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges]\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "#G2,pos,sizes_G2,node_colors_G2 = add_center_circles(xychange,circle_list,tags_that_exist)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiment]\n",
    "\n",
    "nx.draw(G, final_pos_dic, with_labels=True, font_size = 18, font_weight = 'bold',node_size = [n_dic[i]['size']*node_scalar for i in G.nodes], node_color = colors, width = widths)\n",
    "\n",
    "nx.draw_networkx_edges(G, final_pos_dic,edgelist=G.edges,width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in e_dic], alpha=1, edge_color=sentiment_colors)\n",
    "     \n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "plt.title('Sentential Co-Occurrences by\\nLouvain Community (Color), Assort = {}\\n NLP Object (Cluster), Assort = {}'.format(get_att_assort(G,n_dic,'louvain_partition'),get_att_assort(G,n_dic)),size=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TxuFDqQxxo0g"
   },
   "source": [
    "## 7g. Overall Clustering of Sentential Co-Occurences by Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GhD1vLl-Zhjv"
   },
   "outputs": [],
   "source": [
    "louvain_colors = []\n",
    "\n",
    "cmap = cm.get_cmap('Set3')\n",
    "\n",
    "for node in G.nodes:\n",
    "  i = 0\n",
    "  while i < len(louvain_comm_membership):\n",
    "    if node in louvain_comm_membership[i]:\n",
    "      n_dic[node]['louvain_color'] = cmap.colors[i]\n",
    "    i += 1\n",
    "\n",
    "k = 1.3\n",
    "pos_part2 = nx.spring_layout(G,k)\n",
    "pos_part2\n",
    "\n",
    "node_colors = louvain_colors\n",
    "coord_part2 = make_coordinates(len(louvain_comm_membership))\n",
    "colors_that_exist = set(n_dic[i]['louvain_color'] for i in n_dic)\n",
    "colors_that_exist\n",
    "\n",
    "color_pos = []\n",
    "for color in colors_that_exist:\n",
    "  node_list2 = {}\n",
    "  for node in G.nodes:\n",
    "    if n_dic[node]['louvain_color'] == color:\n",
    "      node_list2[node] = pos_part2[node]\n",
    "  color_pos.append(node_list2)\n",
    "color_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j5-uKxW6y-Nj"
   },
   "outputs": [],
   "source": [
    "def circular_color_position(x_list,coord_list,pos,pos_list,n_dic):\n",
    "  cir_pos_list = []\n",
    "  #k = 0.8\n",
    "  i = 0\n",
    "  radius_scalar = 0.25 #2.5/coord_scale\n",
    "  for l in x_list:\n",
    "    G1 = nx.Graph()\n",
    "    new_node_list = []\n",
    "    for node in n_dic:\n",
    "      if n_dic[node]['louvain_color'] == l:\n",
    "        new_node_list.append(node)\n",
    "    G1.add_nodes_from(new_node_list)\n",
    "    #pos = nx.spring_layout(G1, k, iterations=17)\n",
    "    pos = nx.circular_layout(G1,scale=len(G1.nodes)*radius_scalar)\n",
    "    #print(pos)\n",
    "    for key in pos.keys():\n",
    "      pos[key] = [pos[key][0]+coord_list[i][0],pos[key][1]+coord_list[i][1]]\n",
    "    cir_pos_list.append(pos)\n",
    "    i += 1\n",
    "  return cir_pos_list\n",
    "\n",
    "cir_pos_list2 = circular_color_position(colors_that_exist,coord_part2,pos_part2,color_pos,n_dic)\n",
    "circle_list2,rmax2 = show_circle_layout(cir_pos_list2,0.01)\n",
    "xychange2 = find_xydelta(circle_list2,rmax2,3)\n",
    "final_pos_dic2 = centralize_circles(cir_pos_list2,xychange2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qUGQ3AalEATy"
   },
   "outputs": [],
   "source": [
    "def color_by_tag(tags_that_exist,node_dic):\n",
    "  i = 0\n",
    "  cmap = cm.get_cmap('Set3')\n",
    "  node_colors2 = []\n",
    "  for node in node_dic:\n",
    "    i = 0\n",
    "    for tag in tags_that_exist:\n",
    "      if node_dic[node]['tag'] == tag:\n",
    "        node_colors2.append(cmap.colors[i])\n",
    "      i += 1\n",
    "  return node_colors2\n",
    "node_colors2 = color_by_tag(tags_that_exist,n_dic)\n",
    "node_colors2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Br5yzB9scOB0"
   },
   "outputs": [],
   "source": [
    "louvain_pos = {}\n",
    "i = 0\n",
    "for v in sorted(set(louvain_comm_membership)):\n",
    "  key = 'Louvain_Comm_'+ str(v)\n",
    "  louvain_pos[key] = circle_list2[i]\n",
    "  i += 1\n",
    "louvain_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7CTXtbvzLqXL"
   },
   "outputs": [],
   "source": [
    "def overall_louvain_graph(node_dic,edge_dic,louvain_pos,node_scalar=10000,edge_scalar=20):\n",
    "  cmap = cm.get_cmap('Set3')\n",
    "  tag_node_dic = {}\n",
    "  tag_edge_dic = {}\n",
    "  tag_pos_dic = {}\n",
    "  i = 0\n",
    "  for tag in louvain_pos:\n",
    "    node_num = 0\n",
    "    tag_node_dic[tag] = {}\n",
    "    size_l = []\n",
    "    for node in node_dic:\n",
    "      if node_dic[node]['louvain_partition'] == tag:\n",
    "        node_num += 1\n",
    "    tag_node_dic[tag]['node_num'] = node_num\n",
    "    tag_node_dic[tag]['color'] = cmap.colors[i]\n",
    "    tag_pos_dic[tag] = (louvain_pos[tag][0],louvain_pos[tag][1])\n",
    "    tag_node_dic[tag]['radius'] = louvain_pos[tag][2]\n",
    "    i += 1\n",
    "\n",
    "  for tag in louvain_pos:\n",
    "    for tag2 in louvain_pos:\n",
    "      if (tag != tag2) and ((tag2,tag) not in tag_edge_dic.keys()):\n",
    "        sentiment_l = []\n",
    "        width_l = []\n",
    "        for edge in edge_dic:\n",
    "          if (((node_dic[edge[0]]['louvain_partition'] == tag) and (node_dic[edge[1]]['louvain_partition'] ==  tag2)) or ((node_dic[edge[1]]['louvain_partition'] == tag) and (node_dic[edge[0]]['louvain_partition'] ==  tag2))):\n",
    "            sentiment_l.append(edge_dic[edge]['sentiment'])\n",
    "            width_l.append(edge_dic[edge]['width'])\n",
    "        if sentiment_l and width_l:\n",
    "          tag_edge_dic[(tag,tag2)] = {}\n",
    "          tag_edge_dic[(tag,tag2)]['width'] = round(sum(width_l) / len(width_l),2)\n",
    "          tag_edge_dic[(tag,tag2)]['sentiment'] = round(sum(sentiment_l) / len(sentiment_l),2)\n",
    "  #tag_edge_dic = {z: v for z, v in tag_edge_dic.items() if v}\n",
    "  G_all = nx.Graph()\n",
    "  G_all.add_nodes_from(tag_node_dic)\n",
    "  G_all.add_edges_from(tag_edge_dic)\n",
    "\n",
    "  for edge in G_all.edges:\n",
    "    if edge not in tag_edge_dic:\n",
    "      tag_edge_dic[edge] = {}\n",
    "      tag_edge_dic[edge]['width'] = tag_edge_dic[(edge[1],edge[0])]['width']\n",
    "      tag_edge_dic[edge]['sentiment'] = tag_edge_dic[(edge[1],edge[0])]['sentiment']\n",
    "      del tag_edge_dic[(edge[1],edge[0])]\n",
    "\n",
    "  return G_all,tag_node_dic,tag_edge_dic,tag_pos_dic\n",
    "G_all,tag_node_dic,tag_edge_dic,tag_pos_dic = overall_louvain_graph(n_dic,e_dic,louvain_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MLhf_uG_c0wP"
   },
   "outputs": [],
   "source": [
    "from matplotlib.pyplot import cm\n",
    "plt.figure(figsize=(15,15))\n",
    "\n",
    "edge_scalar = 14\n",
    "node_scalar = 20000\n",
    "\n",
    "plt.style.use('seaborn-deep')\n",
    "\n",
    "sentiments = [tag_edge_dic[i]['sentiment'] for i in G_all.edges]\n",
    "node_size = [tag_node_dic[i]['radius']*node_scalar for i in G_all.nodes]\n",
    "colors = [tag_node_dic[i]['color'] for i in G_all.nodes]\n",
    "\n",
    "tag_sentiment = [tag_edge_dic[i]['sentiment'] for i in G_all.edges]\n",
    "widths = [math.log(tag_edge_dic[i]['width']+1)*edge_scalar for i in G_all.edges]\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in tag_sentiment]\n",
    "\n",
    "nx.draw(G_all,tag_pos_dic, with_labels=True, font_size = 18, font_weight = 'bold',alpha=1,node_size=node_size,node_color=colors)\n",
    "\n",
    "nx.draw_networkx_edges(G_all,tag_pos_dic,edgelist=G_all.edges,width=widths,edge_color=sentiment_colors, alpha=1)\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "plt.title('Sentential Co-Occurrence by\\nLouvain Community Summarization (1944)',size=40)\n",
    "plt.xlim(-11.5,12.5)\n",
    "plt.ylim(-15.2,13.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FZG2CA-fD29t"
   },
   "outputs": [],
   "source": [
    "for z,v in louvain_comm_membership.items():\n",
    "   print( 'Louvain Community ' + str(z) + ' contains ' + str(v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dqOaIU3iECEs"
   },
   "outputs": [],
   "source": [
    "for node in G.nodes:\n",
    "  print(node,n_dic[node])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "j9LBHujdTI_x"
   },
   "source": [
    "##7h. Sentential Co-Occurence Clustering by Community "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ro25P7dYFaYt"
   },
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "from mpl_toolkits.axes_grid1 import AxesGrid\n",
    "from matplotlib.pyplot import cm\n",
    "\n",
    "#G,n_dic,e_dic = make_G(df,'sentences',8,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "plt.figure(figsize=(20,15))\n",
    "\n",
    "k = 0.8\n",
    "edge_scalar = 2.5\n",
    "node_scalar = 200\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "cmap2 = cm.get_cmap('Set3')\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "widths = [math.log(e_dic[i]['width']+1)*edge_scalar for i in G.edges]\n",
    "sentiments = [e_dic[i]['sentiment'] for i in G.edges]\n",
    "sizes = [n_dic[i]['size']*node_scalar for i in G.nodes]\n",
    "node_color = [cmap2.colors[tags_that_exist.index(n_dic[i]['tag'])] for i in G.nodes]\n",
    "\n",
    "nx.draw(G, final_pos_dic2, with_labels=True, font_size = 18, font_weight = 'bold',node_size = sizes, node_color = node_color, width = widths)\n",
    "\n",
    "nx.draw_networkx_edges(G, final_pos_dic2,edgelist=G.edges,width=widths, alpha=1, edge_color=sentiments, edge_cmap=cmap)\n",
    "\n",
    "plt.title('Sentential Co-Occurrences in 1944 by\\nNLP Object (Color), Assort = {}\\n Louvain Community (Cluster), Assort = {}'.format(get_att_assort(G,n_dic),get_att_assort(G,n_dic,'louvain_partition')),size=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AsditvBfTSyh"
   },
   "source": [
    "##7i. Egocentric Graph of Diary/Sentential Co-Occurent Terms (Annex/Kitty/Anne)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CRqPbx1iTdLU"
   },
   "outputs": [],
   "source": [
    "plt.subplots(figsize=(15,15))\n",
    "\n",
    "node_scalar = 250\n",
    "edge_scalar = 3\n",
    "\n",
    "ego_name = 'Kitty'\n",
    "\n",
    "Gg,n_dic,e_dic = make_G(df,'sentences',0,node_dic,sent_dic,sentiments2,ego_name,'')\n",
    "\n",
    "#pos = graphviz_layout(G)\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in Gg.edges]\n",
    "\n",
    "if min(sentiments) >= 0:\n",
    "  min_sent = -1\n",
    "else:\n",
    "  min_sent = min(sentiments)\n",
    "\n",
    "if max(sentiments) <= 0:\n",
    "  max_sent = 1\n",
    "else:\n",
    "  max_sent = max(sentiments)\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "cmap2 = cm.get_cmap('Set3')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "color = []\n",
    "i = 0\n",
    "for node in Gg.nodes:\n",
    "  if node == ego_name:\n",
    "    color.append(cmap2.colors[i])\n",
    "  else:\n",
    "    color.append(cmap2.colors[i+1])\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "node_size = []\n",
    "for node in Gg.nodes:\n",
    "  if node != ego_name:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar)\n",
    "  else:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar*1.2)\n",
    "\n",
    "node_and_degree = Gg.degree()\n",
    "(largest_hub, degree) = sorted(node_and_degree, key=itemgetter(1))[-1]\n",
    "# Create ego graph of main hub\n",
    "hub_ego = nx.ego_graph(Gg, largest_hub)\n",
    "# Draw graphu\n",
    "pos = nx.spring_layout(hub_ego)\n",
    "#hub_ego.remove_node(ego_name)\n",
    "nx.draw(hub_ego, pos, node_color=cmap2.colors[4], node_size=node_size, with_labels=True,font_size=12,font_weight='bold',width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in Gg.edges],edge_color=sentiment_colors)\n",
    "# Draw ego as large and red\n",
    "nx.draw_networkx_nodes(hub_ego.remove_node(ego_name), pos, nodelist=[largest_hub], node_size=n_dic[ego_name]['size']*node_scalar, node_color=cmap2.colors[3], with_labels=True,font_size=12,font_weight='bold')\n",
    "\n",
    "plt.title('Egocentric Sentential Co-Occurrence of {}'.format(ego_name),size=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ir6FCdzrWafI"
   },
   "outputs": [],
   "source": [
    "plt.subplots(figsize=(15,15))\n",
    "\n",
    "node_scalar = 250\n",
    "edge_scalar = 3\n",
    "\n",
    "ego_name = 'Anne'\n",
    "\n",
    "Gg,n_dic,e_dic = make_G(df,'sentences',6,node_dic,sent_dic,sentiments2,ego_name,'')\n",
    "\n",
    "#pos = graphviz_layout(G)\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in Gg.edges]\n",
    "\n",
    "if min(sentiments) >= 0:\n",
    "  min_sent = -1\n",
    "else:\n",
    "  min_sent = min(sentiments)\n",
    "\n",
    "if max(sentiments) <= 0:\n",
    "  max_sent = 1\n",
    "else:\n",
    "  max_sent = max(sentiments)\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "cmap2 = cm.get_cmap('Set3')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "color = []\n",
    "i = 0\n",
    "for node in Gg.nodes:\n",
    "  if node == ego_name:\n",
    "    color.append(cmap2.colors[i])\n",
    "  else:\n",
    "    color.append(cmap2.colors[i+1])\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "node_size = []\n",
    "for node in Gg.nodes:\n",
    "  if node != ego_name:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar)\n",
    "  else:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar*1.2)\n",
    "\n",
    "node_and_degree = Gg.degree()\n",
    "(largest_hub, degree) = sorted(node_and_degree, key=itemgetter(1))[-1]\n",
    "# Create ego graph of main hub\n",
    "hub_ego = nx.ego_graph(Gg, largest_hub)\n",
    "# Draw graphu\n",
    "pos = nx.spring_layout(hub_ego,3)\n",
    "#hub_ego.remove_node(ego_name)\n",
    "nx.draw(hub_ego, pos, node_color=cmap2.colors[4], node_size=node_size, with_labels=True,font_size=12,font_weight='bold',width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in Gg.edges],edge_color=sentiment_colors)\n",
    "# Draw ego as large and red\n",
    "nx.draw_networkx_nodes(hub_ego.remove_node(ego_name), pos, nodelist=[largest_hub], node_size=n_dic[ego_name]['size']*node_scalar, node_color=cmap2.colors[3], with_labels=True,font_size=12,font_weight='bold')\n",
    "\n",
    "plt.title('Egocentric Sentential Co-Occurrence of {}'.format(ego_name),size=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EDUS8LIPY7JZ"
   },
   "outputs": [],
   "source": [
    "plt.subplots(figsize=(15,15))\n",
    "\n",
    "node_scalar = 250\n",
    "edge_scalar = 3\n",
    "\n",
    "ego_name = 'Annex'\n",
    "\n",
    "Gg,n_dic,e_dic = make_G(df,'sentences',8,node_dic,sent_dic,sentiments2,ego_name,'')\n",
    "\n",
    "#pos = graphviz_layout(G)\n",
    "\n",
    "sentiments = [e_dic[i]['sentiment'] for i in Gg.edges]\n",
    "\n",
    "if min(sentiments) >= 0:\n",
    "  min_sent = -1\n",
    "else:\n",
    "  min_sent = min(sentiments)\n",
    "\n",
    "if max(sentiments) <= 0:\n",
    "  max_sent = 1\n",
    "else:\n",
    "  max_sent = max(sentiments)\n",
    "\n",
    "cmap = cm.get_cmap('viridis_r')\n",
    "cmap2 = cm.get_cmap('Set3')\n",
    "\n",
    "norm = matplotlib.colors.DivergingNorm(vcenter=0, vmin=min_sent*1.01, vmax=max_sent*1.01)\n",
    "\n",
    "sentiment_colors = [cmap(norm(i)) for i in sentiments]\n",
    "\n",
    "color = []\n",
    "i = 0\n",
    "for node in Gg.nodes:\n",
    "  if node == ego_name:\n",
    "    color.append(cmap2.colors[i])\n",
    "  else:\n",
    "    color.append(cmap2.colors[i+1])\n",
    "\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=min_sent, vmax=max_sent))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm)\n",
    "cbar.ax.set_ylabel('Sentiment',size=30,rotation=270,labelpad=20)\n",
    "\n",
    "node_size = []\n",
    "for node in Gg.nodes:\n",
    "  if node != ego_name:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar)\n",
    "  else:\n",
    "    node_size.append(n_dic[node]['size']*node_scalar*1.2)\n",
    "\n",
    "node_and_degree = Gg.degree()\n",
    "(largest_hub, degree) = sorted(node_and_degree, key=itemgetter(1))[-1]\n",
    "# Create ego graph of main hub\n",
    "hub_ego = nx.ego_graph(Gg, largest_hub)\n",
    "# Draw graphu\n",
    "pos = nx.spring_layout(hub_ego,2)\n",
    "#hub_ego.remove_node(ego_name)\n",
    "nx.draw(hub_ego, pos, node_color=cmap2.colors[4], node_size=node_size, with_labels=True,font_size=12,font_weight='bold',width=[math.log(e_dic[i]['width']+1)*edge_scalar for i in Gg.edges],edge_color=sentiment_colors)\n",
    "# Draw ego as large and red\n",
    "nx.draw_networkx_nodes(hub_ego.remove_node(ego_name), pos, nodelist=[largest_hub], node_size=n_dic[ego_name]['size']*node_scalar, node_color=cmap2.colors[3], with_labels=True,font_size=12,font_weight='bold')\n",
    "\n",
    "plt.title('Egocentric Sentential Co-Occurrence of {}'.format(ego_name),size=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TsaaATw3Z8kH"
   },
   "source": [
    "##8. Graph Partitions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6ITrAbHQ3m7c"
   },
   "source": [
    "###8a. Partition of Fluid Communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_-ttJoFQ38I2"
   },
   "outputs": [],
   "source": [
    "commsd_fl = {}\n",
    "for z,v in color_comm_membership.items():\n",
    "  commsd_fl[z] = len(v)\n",
    "ddf=pd.DataFrame(commsd_fl.items(), columns=['Community', 'Number of Nodes'])\n",
    "sst=\"Bar plot of Fluid Communities\"\n",
    "ddf.plot.bar(x='Community', y='Number of Nodes', figsize=(12,7),rot=0,title=sst);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b6BTH5Hw4-Zv"
   },
   "outputs": [],
   "source": [
    "#G,n_dic,e_dic = make_G(df,'sentences',14,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "node_color = []\n",
    "for node in G.nodes:\n",
    "  for z,v in color_comm_membership.items():\n",
    "    if node in v:\n",
    "      node_color.append(z)\n",
    "\n",
    "vmin = min(node_color) #df['color'].min()\n",
    "vmax = max(node_color) #df['color'].max()\n",
    "cmap=plt.cm.viridis #plt.cm.coolwarm #plt.cm.Blues #\n",
    "\n",
    "colors = [i for i in node_color]\n",
    "\n",
    "plt.figure(figsize=(10,7));\n",
    "node_border_color='k'\n",
    "nodes = nx.draw_networkx_nodes(G, final_pos_dic, node_color=colors, cmap=cmap, vmin=vmin, vmax=vmax)\n",
    "nodes.set_edgecolor(node_border_color)\n",
    "nx.draw_networkx_edges(G, final_pos_dic,edge_color='b',alpha=0.8)\n",
    "plt.axis('off');\n",
    "yoffset = {}\n",
    "y_off = -0.7 #0.05  # offset on the y axis\n",
    "for z, v in final_pos_dic.items():\n",
    "    yoffset[z] = (v[0], v[1]+y_off)\n",
    "nx.draw_networkx_labels(G, yoffset,font_weight='bold',font_size='11');\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm,label=\"Fluid Communities\") #, orientation='horizontal')\n",
    "sst=\"Fluid Communities Clustered by NLP Attribute\"\n",
    "plt.title(sst,fontsize=15);\n",
    "plt.margins(x=0.1, y=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Yajczbs24pTj"
   },
   "source": [
    "###8b. Partition of Lovuain Communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aL9EZUQWZ6M2"
   },
   "outputs": [],
   "source": [
    "ddf=pd.DataFrame(commsd_lo.items(), columns=['Community', 'Number of Nodes'])\n",
    "sst=\"Bar plot of Louvain Communities\"\n",
    "ddf.plot.bar(x='Community', y='Number of Nodes', figsize=(12,7),rot=0,title=sst);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Agcm51RS44Ya"
   },
   "outputs": [],
   "source": [
    "#G,n_dic,e_dic = make_G(df,'sentences',14,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "partition_lo = community.best_partition(G)\n",
    "node_color=partition_lo.values()\n",
    "vmin = min(node_color) #df['color'].min()\n",
    "vmax = max(node_color) #df['color'].max()\n",
    "cmap=plt.cm.viridis #plt.cm.coolwarm #plt.cm.Blues #\n",
    "\n",
    "colors = [i for i in node_color]\n",
    "\n",
    "plt.figure(figsize=(10,7));\n",
    "node_border_color='k'\n",
    "nodes = nx.draw_networkx_nodes(G, final_pos_dic, node_color=colors, cmap=cmap, vmin=vmin, vmax=vmax)\n",
    "nodes.set_edgecolor(node_border_color)\n",
    "nx.draw_networkx_edges(G, final_pos_dic,edge_color='b',alpha=0.8)\n",
    "plt.axis('off');\n",
    "yoffset = {}\n",
    "y_off = -0.7 #0.05  # offset on the y axis\n",
    "for z, v in final_pos_dic.items():\n",
    "    yoffset[z] = (v[0], v[1]+y_off)\n",
    "nx.draw_networkx_labels(G, yoffset,font_weight='bold',font_size='11');\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm,label=\"Louvain Communities\") #, orientation='horizontal')\n",
    "sst=\"Louvain Communities Clustered by NLP Attribute\"\n",
    "plt.title(sst,fontsize=15);\n",
    "plt.margins(x=0.1, y=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Tn2Vi0DI4ucP"
   },
   "source": [
    "###8c. Partition of Leiden Communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PloRm5p8cwbu"
   },
   "outputs": [],
   "source": [
    "ddf=pd.DataFrame(commsd_le.items(), columns=['Community', 'Number of Nodes'])\n",
    "sst=\"Bar plot of Leiden Communities\"\n",
    "ddf.plot.bar(x='Community', y='Number of Nodes', figsize=(12,7),rot=0,title=sst);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b4bpoZTic3F4"
   },
   "outputs": [],
   "source": [
    "#G,n_dic,e_dic = make_G(df,'sentences',14,node_dic,sent_dic,sentiments2,'','Anne')\n",
    "\n",
    "node_color = []\n",
    "for node in G.nodes:\n",
    "  for z,v in leiden_comm_membership.items():\n",
    "    if node in v:\n",
    "      node_color.append(z)\n",
    "\n",
    "vmin = min(node_color) #df['color'].min()\n",
    "vmax = max(node_color) #df['color'].max()\n",
    "cmap=plt.cm.viridis #plt.cm.coolwarm #plt.cm.Blues #\n",
    "\n",
    "for part_list in partition_le:\n",
    "  part = [list(G.nodes())[i] for i in part_list]\n",
    "  partition_nodes.append(part)\n",
    "\n",
    "colors = [i for i in node_color]\n",
    "\n",
    "plt.figure(figsize=(10,7));\n",
    "node_border_color='k'\n",
    "nodes = nx.draw_networkx_nodes(G, final_pos_dic, node_color=colors, cmap=cmap, vmin=vmin, vmax=vmax)\n",
    "nodes.set_edgecolor(node_border_color)\n",
    "nx.draw_networkx_edges(G, final_pos_dic,edge_color='b',alpha=0.8)\n",
    "plt.axis('off');\n",
    "yoffset = {}\n",
    "y_off = -0.7 #0.05  # offset on the y axis\n",
    "for z, v in final_pos_dic.items():\n",
    "    yoffset[z] = (v[0], v[1]+y_off)\n",
    "nx.draw_networkx_labels(G, yoffset,font_weight='bold',font_size='11');\n",
    "sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax))\n",
    "sm.set_array([])\n",
    "cbar = plt.colorbar(sm,label=\"Leiden Communities\") #, orientation='horizontal')\n",
    "sst=\"Leiden Communities Clustered by NLP Attribute\"\n",
    "plt.title(sst,fontsize=15);\n",
    "plt.margins(x=0.1, y=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pB5fWQcRF9CC"
   },
   "source": [
    "###8d. Community Symmetric Differences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YK0L0A5aoku7"
   },
   "outputs": [],
   "source": [
    "def ccomp(A,type1,B,type2):\n",
    "    eds=[]\n",
    "    A1=set(A.keys())\n",
    "    B1=set(B.keys())\n",
    "    for i in A1:\n",
    "        x=set(A[i])\n",
    "        for j in B1:\n",
    "            y=set(B[j])\n",
    "            sd=sorted(list(x.symmetric_difference(y)))\n",
    "            ss=sorted(list(set(list(x)+list(y))))\n",
    "            if sorted(A[i])==sorted(B[j]):\n",
    "                eds.append((type1+str(i),type2+str(j),1.))\n",
    "            else:\n",
    "                if len(sd)/float(len(ss))==1:\n",
    "                    eds.append((type1+str(i),type2+str(j),0.))\n",
    "                else:\n",
    "                    eds.append((type1+str(i),type2+str(j),len(sd)/float(len(ss))))\n",
    "    H = nx.Graph()\n",
    "    H.add_weighted_edges_from(eds)\n",
    "    H.remove_edges_from([e for e in H.edges(data=True) if e[2]['weight']==0.])       \n",
    "    return H"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_6mBkkvMyvUp"
   },
   "outputs": [],
   "source": [
    "fluid_partition = {}\n",
    "for k,v in color_comm_membership.items():\n",
    "  for x in v:\n",
    "    fluid_partition[x] = k\n",
    "\n",
    "type1='Louvain_Comm_'\n",
    "type2='Leiden_Comm_'\n",
    "type3='Fluid_Comm_'\n",
    "H=ccomp(louvain_comm_membership,type1,leiden_comm_membership,type2)\n",
    "H1=ccomp(louvain_comm_membership,type1,color_comm_membership,type3)\n",
    "H2=ccomp(leiden_comm_membership,type2,color_comm_membership,type3)\n",
    "H.add_nodes_from(H1.nodes)\n",
    "H.add_nodes_from(H2.nodes)\n",
    "H.add_weighted_edges_from([(i[0],i[1],H1.edges[i]['weight']) for i in H1.edges])\n",
    "H.add_weighted_edges_from([(i[0],i[1],H2.edges[i]['weight']) for i in H2.edges])\n",
    "\n",
    "edge_scalar = 10\n",
    "\n",
    "posbp={}\n",
    "louvain=sorted([n for n in H.nodes() if \"Louvain\" in n])\n",
    "leiden=sorted([n for n in H.nodes() if \"Leiden\" in n])\n",
    "fluid=sorted([n for n in H.nodes() if \"Fluid\" in n])\n",
    "\n",
    "i=0\n",
    "for node in louvain:\n",
    "  xs = np.linspace(1.0, 5.0, num=len(louvain))\n",
    "  posbp[node] = (xs[i],0)\n",
    "  i += 1\n",
    "\n",
    "i=0\n",
    "for node in leiden:\n",
    "  xs = np.linspace(0.5, 2, num=len(leiden))\n",
    "  posbp[node] = (xs[i],-0.57*xs[i])\n",
    "  i += 1\n",
    "\n",
    "i=0\n",
    "for node in fluid:\n",
    "  xs = np.linspace(4, 6.0, num=len(fluid))\n",
    "  posbp[node] = (xs[i],0.57*xs[i]-3.7)\n",
    "  i += 1\n",
    "\n",
    "elabels={}\n",
    "elabels = nx.get_edge_attributes(H,'weight')\n",
    "elabelsc={}\n",
    "for z,v in elabels.items():\n",
    "    elabelsc[z]='%.02f' %v \n",
    "elabels=elabelsc\n",
    "\n",
    "plt.figure(figsize=(12,10));\n",
    "widths = [float(i) for i in list(elabels.values())]\n",
    "widths = [i/max(widths)*edge_scalar for i in widths]\n",
    "nx.draw(H, posbp, edgelist=H.edges(), edge_color=widths, width=10.0, edge_cmap=plt.cm.Blues)\n",
    "nodes1 = nx.draw_networkx_nodes(H, posbp, nodelist=louvain,node_color=\"b\",node_shape='s',alpha=0.4)\n",
    "nodes2 = nx.draw_networkx_nodes(H, posbp, nodelist=leiden,node_color=\"g\",node_shape='s',alpha=0.4)\n",
    "nodes3 = nx.draw_networkx_nodes(H, posbp, nodelist=fluid,node_color=\"r\",node_shape='s',alpha=0.4)\n",
    "#nx.draw_networkx_edges(H, posbp, edge_colors=widths,width=4, alpha=0.7,edge_cmap=plt.cm.Blues)\n",
    "# nx.draw_networkx_labels(H, posbp)\n",
    "#nx.draw_networkx_edge_labels(H,posbp,edge_labels=elabels,label_pos = 0.7,font_size=7);\n",
    "plt.axis('off');\n",
    "yoffset = {}\n",
    "y_off = -0.05  # offset on the y axis\n",
    "for z, v in posbp.items():\n",
    "    yoffset[z] = (v[0], v[1]+y_off)\n",
    "nx.draw_networkx_labels(H, yoffset,font_weight='bold');\n",
    "plt.margins(x=0.1, y=0.1)\n",
    "plt.title(\"Similarities-Weighted Communities\\nfor the Multilayer Graph (1944)\",size=40);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6yLbktqwHeTA"
   },
   "source": [
    "###8f. Modularity Comparison by Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QbLNoX8d0TA1"
   },
   "outputs": [],
   "source": [
    "modularity_fl=community.modularity(fluid_partition, G, weight='weight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cKnR_BSx1zD5"
   },
   "outputs": [],
   "source": [
    "import networkx.algorithms.community.quality as qu\n",
    "loucomms=[v for k,v in louvain_comm_membership.items()]\n",
    "leicomms=[v for k,v in leiden_comm_membership.items()]\n",
    "fluidcomms=[v for k,v in color_comm_membership.items()]\n",
    "perf_lo=qu.performance(G,loucomms)\n",
    "perf_le=qu.performance(G,leicomms)\n",
    "perf_fl=qu.performance(G,fluidcomms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p1M9VoKF2IU3"
   },
   "outputs": [],
   "source": [
    "cov_lo=qu.coverage(G,loucomms)\n",
    "cov_le=qu.coverage(G,leicomms)\n",
    "cov_fl=qu.coverage(G,fluidcomms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LxrcEkpJ4TcB"
   },
   "outputs": [],
   "source": [
    "plt.style.use('seaborn-dark')\n",
    "\n",
    "labels = ['Louvain','Leiden','Fluid']\n",
    "modularities = [modularity_lo,modularity_le,modularity_fl]\n",
    "performances = [perf_lo,perf_le,perf_fl]\n",
    "coverage = [cov_lo,cov_le,cov_fl]\n",
    "\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "width = 0.25  # the width of the bars\n",
    "buf = 0.1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "rects1 = ax.bar(x - width, modularities, width, label='Modularity')\n",
    "rects2 = ax.bar(x, performances, width, label='Performance')\n",
    "rects3 = ax.bar(x + width, coverage, width, label='Coverage')\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "ax.set_ylabel('Metric Score',size=18)\n",
    "ax.set_title('Community Metrics',size=24)\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels,size=18)\n",
    "ax.legend()\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "fig.set_figheight(10)\n",
    "fig.set_figwidth(8)\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-H8yVvveJVAZ"
   },
   "source": [
    "##9. Cliques"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TKHZ-EirPsLj"
   },
   "source": [
    "###9a. Clique Identification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R_KzYweeLAXO"
   },
   "outputs": [],
   "source": [
    "G_cl,n_dic_cl,e_dic_cl = make_G(df,'sentences',10,node_dic,sent_dic,sentiments2,'','')\n",
    "\n",
    "cliques=[clique for clique in nx.find_cliques(G_cl)]\n",
    "gcliques=[clique for clique in cliques if len(clique)>2]\n",
    "lcliques=[clique for clique in cliques if len(clique)<=2]\n",
    "print(\"The total number of cliques is %i\" %len(cliques))\n",
    "print(\"The total number of non trivial cliques is %i\" %len(gcliques))\n",
    "print(\"The total number of trivial cliques is %i\" %len(lcliques))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yUIH7GPeP7_P"
   },
   "source": [
    "###9b. Bipartite Clique Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "14PNbEeoLsj7"
   },
   "outputs": [],
   "source": [
    "n2cdi={}\n",
    "for n in G_cl.nodes():\n",
    "    for w in gcliques:\n",
    "        if n in w:\n",
    "            n2cdi[n]=w\n",
    "\n",
    "cdi={}\n",
    "for i,j in list(enumerate(cliques)):\n",
    "    if i+1<10:\n",
    "        cdi[\"clique_0\"+str(i+1)]=j\n",
    "    else:\n",
    "        cdi[\"clique_\"+str(i+1)]=j\n",
    "\n",
    "gcdi={k:v for k,v in cdi.items() if len(v)>2}\n",
    "\n",
    "eds=[]\n",
    "for k,v in cdi.items(): #g\n",
    "    for vv in v:\n",
    "        eds.append((k,vv))\n",
    "H=nx.Graph()\n",
    "H.add_edges_from(eds)\n",
    "Y=sorted([n for n in list(H.nodes()) if type(n)==str and \"clique_\" in n])\n",
    "X=sorted([n for n in H.nodes() if n not in Y])\n",
    "\n",
    "pu=1.\n",
    "\n",
    "posbp={}\n",
    "if len(X)==max([len(X),len(Y)]):\n",
    "    for i,x in list(enumerate(X)):\n",
    "        posbp[x]=(0,i)\n",
    "    for j,y in list(enumerate(Y)):\n",
    "        posbp[y]=(1,(j+j*float(i))/len(Y))\n",
    "else:\n",
    "    for i,x in list(enumerate(Y)):\n",
    "        posbp[x]=(1,i)\n",
    "    for j,y in list(enumerate(X)):\n",
    "        posbp[y]=(0,(j+j*float(i))/len(Y))\n",
    "\n",
    "plt.figure(figsize=(7,10));\n",
    "nodes1 = nx.draw_networkx_nodes(H, pos=posbp,node_size=100,nodelist=Y,node_color=\"#b3ffb3\",node_shape='s')\n",
    "nodes2 = nx.draw_networkx_nodes(H, pos=posbp,node_size=100,nodelist=X,node_color=\"#ffb3b3\",node_shape='o')\n",
    "\n",
    "nx.draw_networkx_edges(H, pos=posbp, edge_color=\"b\", alpha=0.5)\n",
    "nx.draw_networkx_labels(H, pos=posbp,font_size=10)\n",
    "plt.axis('off');\n",
    "\n",
    "plt.margins(x=0.25, y=0.1)\n",
    "sst=\"The bipartite graph of nodes vs. cliques\"\n",
    "plt.title(sst);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TrXBpLBQRE7C"
   },
   "source": [
    "###9c. Clique Intersection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_LvI-IGqNGUc"
   },
   "outputs": [],
   "source": [
    "from itertools import combinations\n",
    "\n",
    "cleds=[]\n",
    "# sbeds=[]\n",
    "for k,v in gcdi.items(): \n",
    "    ve=G_cl.subgraph(v).edges()\n",
    "    for w in ve:\n",
    "        cleds.append(w)\n",
    "cleds+=[(c2,c1) for (c1,c2) in cleds]\n",
    "\n",
    "sbeds=[e for e in G_cl.edges() if e not in cleds]\n",
    "\n",
    "\n",
    "ceds=[]\n",
    "for c in list(combinations(Y, 2)):\n",
    "    ints=set(cdi[c[0]]).intersection(cdi[c[1]])\n",
    "    if len(ints)>0:\n",
    "        ceds.append((c[0],c[1],len(ints),list(ints)))\n",
    "\n",
    "weight={(i,j):(k,l) for (i,j,k,l) in ceds}  \n",
    "    \n",
    "w_edges=[(x,y,z) for (x,y),z in weight.items()]\n",
    "\n",
    "cG = nx.Graph()\n",
    "cG.add_weighted_edges_from(w_edges)\n",
    "\n",
    "elabels={}\n",
    "elabels = nx.get_edge_attributes(cG,'weight')\n",
    "edge_width=[cG[u][v]['weight'][0] for u,v in cG.edges()] \n",
    "edge_width=[w for w in edge_width]\n",
    "\n",
    "lcdi={k:len(v) for k,v in cdi.items()}\n",
    "nx.set_node_attributes(cG, lcdi, name=\"clique_size\")\n",
    "\n",
    "nodes_between_2cliques={}\n",
    "tt=[]\n",
    "for (x,y,z) in cG.edges(data=True):\n",
    "    for xx in cdi[x]:\n",
    "        if xx not in n2cdi.keys():\n",
    "            for yy in cdi[y]:\n",
    "                if yy not in n2cdi.keys():\n",
    "                    tt.append((xx,yy,z))\n",
    "for (x,y,z) in tt:\n",
    "    if z['weight'][0]==1:\n",
    "        if z['weight'][1][0] not in n2cdi.keys():\n",
    "#         if lcdi[x]==2 and lcdi[y]==2:\n",
    "            nodes_between_2cliques[(x,y)]=z['weight'][1][0]\n",
    "# print len(set(nodes_between_2cliques.values())),list(set(nodes_between_2cliques.values()))\n",
    "\n",
    "figsize=(15,15)\n",
    "pos=graphviz_layout(cG); #pos=nx.circular_layout(G); \n",
    "node_color=\"#b3ffb3\" #\"#ffb3b3\"\n",
    "node_border_color=\"g\"\n",
    "edge_color=\"#668cff\"\n",
    "plt.figure(figsize=figsize);\n",
    "nodes = nx.draw_networkx_nodes(cG, pos, node_size=[100*v for v in lcdi.values()],node_color=node_color,node_shape=\"s\")\n",
    "nodes.set_edgecolor(node_border_color)\n",
    "nx.draw_networkx_edges(cG, pos, edge_color=edge_color,width=edge_width)\n",
    "nx.draw_networkx_labels(cG, pos)\n",
    "# nx.draw_networkx_edge_labels(cG,pos,edge_labels=elabels);\n",
    "plt.axis('off');\n",
    "sst=\"The intersection graph of cliques\"\n",
    "plt.title(sst);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "E5AQT7__RjDH"
   },
   "source": [
    "###9d. Clustering of Clique Intersection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JWqA73H0NS6B"
   },
   "outputs": [],
   "source": [
    "intersecting_clique_edges=[]\n",
    "for e in cG.edges(data=True):\n",
    "    a=cdi[e[0]]\n",
    "    b=cdi[e[1]]\n",
    "    c=set(a).intersection(set(b))\n",
    "    if len(c)>1:\n",
    "        sG=G_cl.subgraph(list(c))\n",
    "        for se in sG.edges():\n",
    "            if se not in intersecting_clique_edges:\n",
    "                intersecting_clique_edges.append(se)\n",
    "\n",
    "leaves=[n for n in G_cl.nodes() if G_cl.degree(n)==1]\n",
    "\n",
    "pendants=[e for e in G_cl.edges() if e[0] in leaves or e[1] in leaves]\n",
    "\n",
    "nb2c=list(set(nodes_between_2cliques.values()))\n",
    "\n",
    "pos=graphviz_layout(G_cl,'neato')\n",
    "plt.figure(figsize=(15,15));\n",
    "\n",
    "nx.draw_networkx_edges(G_cl, pos, edgelist=[e for e in sbeds if e not in pendants],width=3,edge_color=\"r\", alpha=0.7)\n",
    "nx.draw_networkx_edges(G_cl, pos, edgelist=pendants,width=3,edge_color=\"orange\", alpha=0.5)\n",
    "nx.draw_networkx_edges(G_cl, pos, edgelist=[e for e in G_cl.edges() if e not in sbeds and e not in pendants and e not in intersecting_clique_edges],width=3,edge_color=\"c\", alpha=0.7)\n",
    "nx.draw_networkx_edges(G_cl, pos, edgelist=intersecting_clique_edges,width=3,edge_color=\"b\", alpha=0.8)\n",
    "nodes = nx.draw_networkx_nodes(G_cl, pos, nodelist=[n for n in G_cl.nodes() if n not in nb2c and n not in leaves],node_size=500,node_color=\"r\", alpha=0.9)\n",
    "nodes = nx.draw_networkx_nodes(G_cl, pos, nodelist=nb2c,node_size=500,node_color=\"g\", alpha=0.9)\n",
    "nodes = nx.draw_networkx_nodes(G_cl, pos, nodelist=leaves,node_size=500,node_color=\"orange\", alpha=0.9)\n",
    "\n",
    "plt.axis('off');\n",
    "yoffset = {}\n",
    "y_off = - 15\n",
    "for k, v in pos.items():\n",
    "    yoffset[k] = (v[0], v[1]+y_off)\n",
    "nx.draw_networkx_labels(G_cl, yoffset,font_weight='bold',font_size=13);\n",
    "plt.margins(x=0.25, y=0.2)\n",
    "sst=\"Cliques of Sentential Co-Occurrence (1944)\"#\\n green nodes = brokers between cliques \\n orange nodes = leaves \\n red edges = bridges between cliques \\n cyan edges = non-intersecting clique edges \\n blue edges = intersecting clique edges\"\n",
    "plt.title(sst,size=35);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NDRn0GLPSBWB"
   },
   "source": [
    "##10. Network Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uQ-4Iyo3SLVe"
   },
   "source": [
    "##10a. Centralities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n6HBRP1yb97r"
   },
   "outputs": [],
   "source": [
    "def create_centralities_list(G_cl,maxiter=2000,pphi=5,centList=[]):\n",
    "    if len(centList)==0:\n",
    "        centList=['degree','closeness','betweenness','eigenvector','Katz','PageRank','HITS','load','communicability','current flow']\n",
    "    cenLen=len(centList)\n",
    "    valus={}\n",
    "    for uu,centr in enumerate(centList):\n",
    "        if centr=='degree':\n",
    "            if isinstance(G_cl,nx.DiGraph):\n",
    "                cent=nx.in_degree_centrality(G_cl)\n",
    "                sstt='In Degree Centralities '\n",
    "                valus['in_degree']=cent\n",
    "                cent=nx.out_degree_centrality(G_cl)\n",
    "                sstt+= 'and Out Degree Centralities'\n",
    "                valus['out_degree']=cent\n",
    "            else:\n",
    "                cent=nx.degree_centrality(G_cl)\n",
    "                sstt='Degree Centralities'\n",
    "                ssttt='degree centrality'\n",
    "                valus[centr]=cent\n",
    "        elif centr=='closeness':\n",
    "            cent=nx.closeness_centrality(G_cl)\n",
    "            sstt='Closeness Centralities'\n",
    "            ssttt='closeness centrality'\n",
    "            valus[centr]=cent\n",
    "        elif centr =='load':\n",
    "            cent=nx.load_centrality(G_cl)\n",
    "            sstt='Load Centraities'\n",
    "            valus[centr]=cent\n",
    "        elif centr == 'communicability':\n",
    "            if not isinstance(G_cl, nx.DiGraph):\n",
    "                cent=nx.communicability_betweenness_centrality(G_cl)\n",
    "                sstt='Communicability Centralities'\n",
    "                valus[centr]=cent\n",
    "        elif centr=='betweenness':\n",
    "            cent=nx.betweenness_centrality(G_cl)\n",
    "            sstt='Betweenness Centralities'\n",
    "            ssttt='betweenness centrality'\n",
    "            valus[centr]=cent\n",
    "        elif centr=='current flow':\n",
    "            if not isinstance(G_cl, nx.DiGraph):\n",
    "            \n",
    "                cent=nx.current_flow_closeness_centrality(G_cl)\n",
    "                sstt='Current Flow Closeness Centrality'\n",
    "                valus[centr]=cent\n",
    "        elif centr=='eigenvector':\n",
    "            try:\n",
    "                cent=nx.eigenvector_centrality(G_cl,max_iter=maxiter)\n",
    "                sstt='Eigenvector Centralities'\n",
    "                ssttt='eigenvector centrality'\n",
    "                valus[centr]=cent\n",
    "\n",
    "            except:\n",
    "                valus[centr]=None\n",
    "\n",
    "                continue\n",
    "        elif centr=='Katz':\n",
    "            phi = (1+math.sqrt(pphi))/2.0 # largest eigenvalue of adj matrix\n",
    "            cent=nx.katz_centrality_numpy(G_cl,1/phi-0.01)\n",
    "            cent=nx.katz_centrality_numpy(G_cl,.05)#,1/phi-0.01)\n",
    "            \n",
    "            sstt='Katz Centralities'\n",
    "            ssttt='Katz centrality'\n",
    "            valus[centr]=cent\n",
    "#             valus[centr+'_%i' %pphi]=cent\n",
    "\n",
    "        elif centr=='PageRank':\n",
    "            try:\n",
    "                cent=nx.pagerank(G_cl)\n",
    "                sstt='PageRank'\n",
    "                ssttt='pagerank'\n",
    "                valus[centr]=cent\n",
    "\n",
    "            except:\n",
    "                valus[centr]=None\n",
    "\n",
    "                continue\n",
    "        elif centr=='HITS':\n",
    "            if isinstance(G_cl,nx.DiGraph):\n",
    "                dd=nx.hits(G_cl,max_iter=maxiter)\n",
    "                sstt='HITS hubs '\n",
    "                valus['HITS_hubs']=dd[0]\n",
    "                sstt+= 'and HITS authorities'\n",
    "                valus['HITS_auths']=dd[1]\n",
    "            else:\n",
    "                dd=nx.hits(G_cl,max_iter=maxiter)\n",
    "                cent=nx.degree_centrality(G_cl)\n",
    "                sstt='HITS'\n",
    "                ssttt='HITS Centralities'\n",
    "                valus[centr]=dd[0]\n",
    "        else:\n",
    "            continue\n",
    "        print('%s done!!!' %sstt)\n",
    "    return valus\n",
    "\n",
    "dindices=['out_degree','in_degree','closeness','betweenness','eigenvector','HITS_hubs','HITS_auths','Katz','PageRank','load']\n",
    "indices=['degree','closeness','betweenness','eigenvector','HITS','Katz','PageRank','load','communicability','current flow']\n",
    "# indices=['degree','closeness','betweenness','eigenvector']\n",
    "\n",
    "# Without 'communicability' and 'current flow' (undirected case)\n",
    "dindicesd=['out_degree','in_degree','closeness','betweenness','eigenvector','HITS_hubs','HITS_auths','Katz','PageRank','load']\n",
    "indicesd=['degree','closeness','betweenness','eigenvector','HITS','Katz','PageRank','load']\n",
    "# indicesd=['degree','closeness','betweenness','eigenvector']\n",
    "\n",
    "dindicesdr=dindices\n",
    "indicesdr=indices\n",
    "\n",
    "# Plus 'node'\n",
    "dindicesdrn=[\"node\"]+dindices\n",
    "indicesdrn=['node']+indices\n",
    "\n",
    "def picker(G_cl,data,cols,thres):\n",
    "    for al in range(1,len(G_cl.nodes())+1):\n",
    "        degs_dic={}\n",
    "        deggs=None\n",
    "        ndegs_dic={}\n",
    "        for col in cols:\n",
    "            if col not in ndegs_dic:\n",
    "                ndegs_dic[col]={}\n",
    "            degress=central_pd.sort_values(col,ascending=False).head(al).to_dict()\n",
    "            if deggs is None:\n",
    "                deggs=set(degress[col].keys())\n",
    "            else:\n",
    "                deggs=deggs.intersection(set(degress[col].keys()))\n",
    "            degs_dic[col]=degress[col]#.keys()\n",
    "            for ije, vvf in enumerate(sorted(set(degress[col].values()),reverse=True)):\n",
    "                ndegs_dic[col][vvf]=ije+1\n",
    "        if len(deggs)>=thres:\n",
    "            break\n",
    "\n",
    "    rank_dic=Counter()\n",
    "    rrank_dic={}\n",
    "    for i in deggs:\n",
    "        ss='Key-player node %s is ranked: \\n' %i  #has ranking index\n",
    "        if i not in rrank_dic:\n",
    "            rrank_dic[i]=[]\n",
    "        for nn,kk in degs_dic.items():\n",
    "            ss+='%i-th wrt %s (%f)\\n' %(ndegs_dic[nn][kk[i]],nn,kk[i])#(kk.index(i)+1,nn)\n",
    "            rank_dic[i]+=ndegs_dic[nn][kk[i]]\n",
    "            rrank_dic[i].append(ndegs_dic[nn][kk[i]])\n",
    "#         print ss\n",
    "    return al,deggs,degs_dic,rank_dic,rrank_dic,ndegs_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9TjmdR4hZ6Df"
   },
   "outputs": [],
   "source": [
    "central_pd=pd.DataFrame(create_centralities_list(G_cl))\n",
    "central_pd[\"node\"]=central_pd.index.values\n",
    "if isinstance(G_cl,nx.DiGraph):\n",
    "    central_pd=central_pd[['node']+dindices]\n",
    "else:\n",
    "    central_pd=central_pd[['node']+indices]\n",
    "central_pd[\"node\"]=central_pd.index.values\n",
    "central_pd.reset_index(drop = True, inplace = True)\n",
    "central_pd.sort_values('node') #.head()\n",
    "central_pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mkIT10g1jH8Q"
   },
   "outputs": [],
   "source": [
    "data = central_pd\n",
    "\n",
    "if isinstance(G_cl,nx.DiGraph):\n",
    "#     cols=dindicesdr# no hits\n",
    "    cols=dindices\n",
    "else:\n",
    "#     cols=indicesdr#no hits\n",
    "    cols=indices\n",
    "    \n",
    "thres=len(G_cl.nodes())\n",
    "\n",
    "# al,deggs,degs_dic,rankings,rankd = picker(G,data,cols,thres) #,thres=1)\n",
    "al,deggs,degs_dic,rankings,rrank_dic,ndegs_dic = picker(G_cl,data,cols,thres) #,thres=1)\n",
    "\n",
    "central_pd=data.set_index('node')\n",
    "nndegs_dic={}\n",
    "ppdff=central_pd.to_dict(orient='dict')\n",
    "\n",
    "for k,v in ppdff.items():\n",
    "    nndegs_dic[k]=[ndegs_dic[k][j] for j in v.values()]\n",
    "    \n",
    "nndegs_dic['node']=[j.keys() for j in ppdff.values()][0]\n",
    "\n",
    "fpdngegs=pd.DataFrame.from_dict(nndegs_dic, orient='index')\n",
    "fpdngegs=fpdngegs.T\n",
    "fpdngegs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tIW21bDXjW_Q"
   },
   "outputs": [],
   "source": [
    "if isinstance(G_cl,nx.DiGraph):\n",
    "    df_c=fpdngegs[dindices]\n",
    "else:\n",
    "    df_c=fpdngegs[indices]\n",
    "    \n",
    "rd={}\n",
    "for i in df_c.index.values:\n",
    "    vv=df_c.iloc[i].tolist()\n",
    "    ra=min(vv)\n",
    "    vv=len(df_c.columns)*[ra]\n",
    "    if isinstance(G_cl,nx.DiGraph):\n",
    "        dff=df_c[(df_c['out_degree'] >= vv[0]) & (df_c['in_degree'] >= vv[1]) & (df_c['closeness'] >= vv[2]) & (df_c['betweenness'] >= vv[3]) & (df_c['eigenvector'] >= vv[4]) & (df_c['HITS_hubs'] >= vv[5]) & (df_c['HITS_auths'] >= vv[6]) & (df_c['Katz'] >= vv[7]) & (df_c['PageRank'] >= vv[8]) & (df_c['load'] >= vv[9])]   \n",
    "    else:\n",
    "        dff=df_c[(df_c['degree'] >= vv[0]) & (df_c['closeness'] >= vv[1]) & (df_c['betweenness'] >= vv[2]) & (df_c['eigenvector'] >= vv[3]) & (df_c['Katz'] >= vv[4]) & (df_c['PageRank'] >= vv[5]) & (df_c['load'] >= vv[6]) & (df_c['communicability'] >= vv[7]) & (df_c['current flow'] >= vv[8])]             \n",
    "    rd[i]=len(dff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FP-TCPs9jZsR"
   },
   "outputs": [],
   "source": [
    "mrd={}\n",
    "for i,v in rd.items():\n",
    "    mrd[fpdngegs.iloc[i][\"node\"]]=max(rd.values())-rd[i]+1\n",
    "mrd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DKcdAziTmO90"
   },
   "outputs": [],
   "source": [
    "nn=[]\n",
    "rn=[]\n",
    "ru=[]\n",
    "rankings=rd\n",
    "en=enumerate(sorted(rankings, key=rankings.get, reverse=False))\n",
    "r = {key: rank for rank, key in enumerate(sorted(set(rankings.values()), reverse=False), 1)}\n",
    "rdd={k: r[v] for k,v in rankings.items()}\n",
    "for i, w in en:\n",
    "    nn.append(fpdngegs.iloc[w]['node'])  \n",
    "    rn.append(max(rdd.values())-rdd[w]+1) \n",
    "    ru.append(rankings[w])\n",
    "rdf=pd.DataFrame(\n",
    "    {'node': nn, \n",
    "     'rung' : ru,\n",
    "     'rank': rn\n",
    "    })\n",
    "rdf=rdf[[\"node\",\"rung\",\"rank\"]]\n",
    "rdf.sort_values(\"rank\", inplace=True)\n",
    "rdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ap2u9d0wmaFn"
   },
   "outputs": [],
   "source": [
    "rdf[indices]=fpdngegs[indices]\n",
    "import seaborn as sns\n",
    "mrdf = rdf\n",
    "cm = sns.light_palette(\"green\", as_cmap=True)\n",
    "mrdf[['degree','closeness','betweenness','eigenvector','HITS','Katz','PageRank','load','communicability','current flow']] = mrdf[['degree','closeness','betweenness','eigenvector','HITS','Katz','PageRank','load','communicability','current flow']].apply(pd.to_numeric)\n",
    "mrdf.style.background_gradient(cmap='viridis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rUEcSHQdmfnm"
   },
   "outputs": [],
   "source": [
    "clr={}\n",
    "for i in range(len(rdf)):\n",
    "    clr[rdf.iloc[i][\"node\"]]=rdf.iloc[i][\"rank\"]\n",
    "node_color=[clr[n] for n in G_cl.nodes()]   \n",
    "\n",
    "nsi=[50*G_cl.degree(n) for n in G_cl.nodes()]\n",
    "\n",
    "labels={}\n",
    "for i in G_cl.nodes():\n",
    "    labels[i]=i\n",
    "\n",
    "edge_scalar = 2\n",
    "node_scalar = 3\n",
    "nsi = [i*node_scalar for i in nsi]\n",
    "edge_width = [math.log(e_dic_cl[i]['width']+1)*edge_scalar for i in G_cl.edges]\n",
    "\n",
    "plt.figure(figsize=(18,10))\n",
    "nx.draw_networkx_edges(G_cl,pos=pos,width=edge_width, edge_color=\"lime\", alpha=0.8);\n",
    "nc=nx.draw_networkx_nodes(G_cl,pos=pos,cmap=plt.get_cmap('rainbow'), node_color=node_color,edgecolors='r',node_size=nsi,alpha=1);\n",
    "nx.draw_networkx_labels(G_cl,pos=pos,labels=labels,font_size=12);\n",
    "\n",
    "cbar=plt.colorbar(nc)\n",
    "#cbar.set_label('Aggregated Centralities Rating')\n",
    "cbar.ax.set_ylabel('Aggregated Centralities Rating',size=20,rotation=270,labelpad=30)\n",
    "plt.axis(\"off\");\n",
    "tt=\"Sentential Co-Occurrence\\nNodes Colored According to Aggregated Centrality Rating\"\n",
    "plt.title(tt,size=25);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "B8r-csvMJT6Y"
   },
   "outputs": [],
   "source": [
    "for l in sent_list:\n",
    "  if ('van D.' in l) and ('Boche' in l):\n",
    "    print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VvF30wn-WgtG"
   },
   "outputs": [],
   "source": [
    "name = 'Sentential Co-Occurrences'\n",
    "\n",
    "df4 = df.loc[ list(G_cl.nodes) , list(G_cl.nodes) ].copy()#df3[list(node_dic.keys())].copy()\n",
    "axes = pd.plotting.scatter_matrix(df4, figsize=(15, 15), marker='o',hist_kwds={'bins': 20}, s=60, alpha=.8)\n",
    "corr = df4.corr().values \n",
    "for i, j in zip(*plt.np.triu_indices_from(axes, k=1)):\n",
    "    axes[i, j].annotate(\"%.3f\" %corr[i,j], (0.8, 0.8), xycoords='axes fraction', ha='left', va='center',rotation='vertical')\n",
    "sst=\"%s \\n Scatter Matrix of Centrality Indices \\n (displaying Pearson correlation coefficients)\" %name\n",
    "\n",
    "n = len(G_cl.nodes)\n",
    "\n",
    "for x in range(n):\n",
    "    for y in range(n):\n",
    "        # to get the axis of subplots\n",
    "        ax = axes[x, y]\n",
    "        # to make x axis name vertical  \n",
    "        ax.xaxis.label.set_rotation(90)\n",
    "        # to make y axis name horizontal \n",
    "        ax.yaxis.label.set_rotation(0)\n",
    "        # to make sure y axis names are outside the plot area\n",
    "        ax.yaxis.labelpad = 20\n",
    "\n",
    "plt.suptitle(sst,fontsize=20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7fptSA96JQjb"
   },
   "outputs": [],
   "source": [
    "name = 'Sentential Co-Occurrence'\n",
    "pos=graphviz_layout(G_cl,'circo')\n",
    "valus=create_centralities_list(G_cl,maxiter=2000,pphi=5,centList=[])\n",
    "if isinstance(G_cl,nx.DiGraph):\n",
    "    cts=['out_degree','in_degree','closeness','betweenness','eigenvector','HITS_hubs','HITS_auths','Katz','PageRank','load']\n",
    "else:\n",
    "    cts=['degree','closeness','betweenness','eigenvector','HITS','Katz','PageRank','load','communicability','current flow']\n",
    "for cent in valus.values():\n",
    "    cs={}\n",
    "    for k,v in cent.items():\n",
    "        if v not in cs:\n",
    "            cs[v]=[k]\n",
    "        else:\n",
    "            cs[v].append(k)\n",
    "    nodrank=[]\n",
    "    uui=0\n",
    "    for k in sorted(cs,reverse=True):\n",
    "        for v in cs[k]:\n",
    "            if uui<len(G_cl): #5:\n",
    "                nodrank.append(v)\n",
    "                uui+=1\n",
    "    nodeclo=[]\n",
    "    for k,v in cent.items():\n",
    "        if k in  nodrank :\n",
    "            nodeclo.append(v)\n",
    "        else:\n",
    "            nodeclo.append(0.)\n",
    "plt.figure(figsize=(25,50))\n",
    "for i,j in enumerate(cts): \n",
    "    plt.subplot(len(cts)/2,2,i+1).set_title(j)\n",
    "    nx.draw_networkx_nodes(G_cl,pos=pos,nodelist=valus[j].keys(),\n",
    "                               node_size=[500*x for x in valus[j].values()],\n",
    "                               node_color=nodeclo,\n",
    "                               cmap=plt.cm.cool,alpha=0.7) #Reds\n",
    "    nx.draw_networkx_edges(G_cl,pos=pos,edge_color='b', alpha=0.5)\n",
    "    y_off = 50 #0.05  # offset on the y axis\n",
    "    pos_off = {key: (pos[key][0],pos[key][1] - y_off) for key in pos}\n",
    "    nx.draw_networkx_labels(G_cl,pos=pos_off,font_weight='bold',font_size='10');\n",
    "    nx.draw_networkx_edges(G_cl,pos=pos,edge_color='b', alpha=0.5)\n",
    "    plt.title(j,fontsize=20)\n",
    "    kk=plt.axis('off')\n",
    "sst=\"%s Centrality Indices\" %name\n",
    "plt.suptitle(sst,fontsize=20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vsUGsmmq1ugY"
   },
   "outputs": [],
   "source": [
    "comm_dic = nx.communicability(G_cl)\n",
    "comm_dic2 = {}\n",
    "for k,z in comm_dic.items():\n",
    "  comm_dic2[k] = {}\n",
    "  comm_dic2[k]['communicability'] = {}\n",
    "  data_list = []\n",
    "  for k1,z1 in z.items():\n",
    "    data_list.append(z1)\n",
    "  comm_dic2[k]['communicability'] = round(sum(data_list)/len(data_list),3)\n",
    "  sent_avg = []\n",
    "  widths2 = []\n",
    "  for e in e_dic_cl.keys():\n",
    "    if (k in e[0]) or (k in e[1]):\n",
    "      sent_avg.append(e_dic_cl[e]['sentiment']*e_dic_cl[e]['width'])\n",
    "      widths2.append(e_dic_cl[e]['width'])\n",
    "  comm_dic2[k]['avg_sentiment'] = round(sum(sent_avg)/(sum(widths2)*len(sent_avg)),3)\n",
    "comm_dic2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oQqI9ynGKTEd"
   },
   "outputs": [],
   "source": [
    "yd = [comm_dic2[k]['avg_sentiment'] for k,z in comm_dic2.items()]\n",
    "xd = [comm_dic2[k]['communicability'] for k,z in comm_dic2.items()]\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "# make the scatter plot\n",
    "plt.scatter(xd, yd, s=30, alpha=0.75, marker='o')\n",
    "\n",
    "# determine best fit line\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "# coefficient of determination, plot text\n",
    "variance = np.var(yd)\n",
    "residuals = np.var([(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)])\n",
    "Rsqr = np.round(1-residuals/variance, decimals=2)\n",
    "plt.text(.4*max(xd)+.1*min(xd),0.2*max(yd)+.1*min(yd),'$R^2 = %0.2f$'% Rsqr, fontsize=30)\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Communicability\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Communicability',size=25)\n",
    "\n",
    "# error bounds\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "# yerrUpper = [(xx*slope+intercept)+(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "# yerrLower = [(xx*slope+intercept)-(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "\n",
    "for k,z in comm_dic.items():\n",
    "  y,x = comm_dic2[k]['avg_sentiment'],comm_dic2[k]['communicability']\n",
    "  if (abs(y) > 0.08) or (k=='Dutch'):\n",
    "    plt.text(x+.01,y+.02, k, fontsize=12,fontweight='bold')\n",
    "\n",
    "plt.plot(xl, yl, '-r')\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "# plt.plot(xd, yerrLower, '--r')\n",
    "# plt.plot(xd, yerrUpper, '--r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oA_gV35jMwhf"
   },
   "outputs": [],
   "source": [
    "xd_list = [i/(len(G.nodes)*len(G.edges)) for i in xd]\n",
    "xd_list = [(i-min(xd_list)) for i in xd_list]\n",
    "#xd_list = xd\n",
    "yd_list = yd\n",
    "xd_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PkR7PqOpNFb9"
   },
   "outputs": [],
   "source": [
    "import scipy\n",
    "scipy.stats.pearsonr(xd, yd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zG6qXmhzNjwy"
   },
   "outputs": [],
   "source": [
    "clos_dic = nx.closeness_centrality(G_cl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i2hsax-EuR9I"
   },
   "outputs": [],
   "source": [
    "yd = [comm_dic2[k]['avg_sentiment'] for k,z in clos_dic.items()]\n",
    "xd = [clos_dic[k] for k,z in clos_dic.items()]\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "# make the scatter plot\n",
    "plt.scatter(xd, yd, s=30, alpha=0.75, marker='o')\n",
    "\n",
    "# determine best fit line\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "# coefficient of determination, plot text\n",
    "variance = np.var(yd)\n",
    "residuals = np.var([(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)])\n",
    "Rsqr = np.round(1-residuals/variance, decimals=2)\n",
    "plt.text(.7*max(xd)+.1*min(xd),0.25*max(yd)+.1*min(yd),'$R^2 = %0.2f$'% Rsqr, fontsize=30)\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Closeness\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Closeness',size=25)\n",
    "\n",
    "# error bounds\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "# yerrUpper = [(xx*slope+intercept)+(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "# yerrLower = [(xx*slope+intercept)-(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "\n",
    "for k,z in clos_dic.items():\n",
    "  y,x = comm_dic2[k]['avg_sentiment'],clos_dic[k]\n",
    "  if (abs(y) > 0.08) or k=='Dutch':\n",
    "    plt.text(x+.005,y+.025, k, fontsize=12,fontweight='bold')\n",
    "\n",
    "plt.plot(xl, yl, '-r')\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "# plt.plot(xd, yerrLower, '--r')\n",
    "# plt.plot(xd, yerrUpper, '--r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MfeHedUmN2hJ"
   },
   "outputs": [],
   "source": [
    "xd = [(i-min(xd)) for i in xd]\n",
    "xd_list = xd_list + xd\n",
    "yd_list = yd_list + yd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H-J7DpwJNTq3"
   },
   "outputs": [],
   "source": [
    "load_dic = nx.load_centrality(G_cl,normalized=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JoNBTTgM2VpS"
   },
   "outputs": [],
   "source": [
    "load_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sgdcEGdHNWDO"
   },
   "outputs": [],
   "source": [
    "yd = [comm_dic2[k]['avg_sentiment'] for k,z in load_dic.items()]\n",
    "xd = [load_dic[k] for k,z in load_dic.items()]\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "# make the scatter plot\n",
    "plt.scatter(xd, yd, s=30, alpha=0.75, marker='o')\n",
    "\n",
    "# determine best fit line\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "# coefficient of determination, plot text\n",
    "variance = np.var(yd)\n",
    "residuals = np.var([(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)])\n",
    "Rsqr = np.round(1-residuals/variance, decimals=2)\n",
    "plt.text(.5*max(xd)+.1*min(xd),0.25*max(yd)+.1*min(yd),'$R^2 = %0.2f$'% Rsqr, fontsize=30)\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Load\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Load',size=25)\n",
    "\n",
    "# error bounds\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "# yerrUpper = [(xx*slope+intercept)+(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "# yerrLower = [(xx*slope+intercept)-(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "\n",
    "for k,z in load_dic.items():\n",
    "  y,x = comm_dic2[k]['avg_sentiment'],load_dic[k]\n",
    "  if (abs(y) > 0.08) and k!='French' or k=='Dutch':\n",
    "    plt.text(x+0.003,y+.0025, k, fontsize=10,fontweight='bold')\n",
    "  if (abs(y) > 0.08) and k=='French':\n",
    "    plt.text(x+0.003,y-.015, k, fontsize=10,fontweight='bold')\n",
    "\n",
    "plt.plot(xl, yl, '-r')\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "# plt.plot(xd, yerrLower, '--r')\n",
    "# plt.plot(xd, yerrUpper, '--r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y4hSgCsxQYpf"
   },
   "outputs": [],
   "source": [
    "xd_list = xd_list + xd\n",
    "yd_list = yd_list + yd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HzWKavJ5QZc2"
   },
   "outputs": [],
   "source": [
    "from matplotlib.pyplot import cm\n",
    "def chunks(lst, n):\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]\n",
    "\n",
    "xlst=xd_list\n",
    "ylst=yd_list\n",
    "n=int(len(xd_list)/3)\n",
    "xd_list1 = [xlst[i:i + n] for i in range(0, len(xlst), n)]\n",
    "yd_list1 = [ylst[i:i + n] for i in range(0, len(ylst), n)]\n",
    "\n",
    "max_val = max([max(i) for i in xd_list1])\n",
    "\n",
    "i = 0\n",
    "while i < len(xd_list1):\n",
    "  this_max = max(xd_list1[i])\n",
    "  if this_max != max_val:\n",
    "    xd_list1[i] = [j*max_val/this_max for j in xd_list1[i]]\n",
    "  i += 1\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "arr = []\n",
    "colors = cm.viridis_r(np.linspace(0, 1, len(yd_list1)))\n",
    "\n",
    "j_ind = list(load_dic.keys()).index('Jew')\n",
    "\n",
    "i = 0\n",
    "s_sizes = [400,200,75]\n",
    "for x,y,c in zip(xd_list1,yd_list1, colors):\n",
    "  arr.append(plt.scatter(x, y, color=c,marker='o',s=s_sizes[i]))\n",
    "  i += 1\n",
    "\n",
    "xdj_list1 = [el[j_ind] for el in xd_list1]\n",
    "ydj_list1 = [el[j_ind] for el in yd_list1]\n",
    "\n",
    "i = 0\n",
    "for x,y,c in zip(xdj_list1,ydj_list1, colors):\n",
    "  arr.append(plt.scatter(x, y, color=c,marker='X',s=s_sizes[i]*4.5))\n",
    "  i += 1\n",
    "\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Relative Centrality\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Centrality',size=25)\n",
    "\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "plt.legend(arr,('Communicability','Closeness','Load'),prop={'size': 15})\n",
    "\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9NLq2IZ8Fl_J"
   },
   "outputs": [],
   "source": [
    "yd = [comm_dic2[k]['avg_sentiment'] for k,z in load_dic.items()]\n",
    "xd = [load_dic[k] for k,z in load_dic.items()]\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "# make the scatter plot\n",
    "plt.scatter(xd, yd, s=30, alpha=0.75, marker='o')\n",
    "\n",
    "# determine best fit line\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "# coefficient of determination, plot text\n",
    "variance = np.var(yd)\n",
    "residuals = np.var([(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)])\n",
    "Rsqr = np.round(1-residuals/variance, decimals=2)\n",
    "plt.text(.5*max(xd)+.1*min(xd),0.25*max(yd)+.1*min(yd),'$R^2 = %0.2f$'% Rsqr, fontsize=30)\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Load\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Load',size=25)\n",
    "\n",
    "# error bounds\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "# yerrUpper = [(xx*slope+intercept)+(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "# yerrLower = [(xx*slope+intercept)-(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "\n",
    "for k,z in load_dic.items():\n",
    "  y,x = comm_dic2[k]['avg_sentiment'],load_dic[k]\n",
    "  if (abs(y) > 0.08) and k!='French' or k=='Dutch':\n",
    "    plt.text(x+0.003,y+.0025, k, fontsize=10,fontweight='bold')\n",
    "  if (abs(y) > 0.08) and k=='French':\n",
    "    plt.text(x+0.003,y-.015, k, fontsize=10,fontweight='bold')\n",
    "\n",
    "plt.plot(xl, yl, '-r')\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "# plt.plot(xd, yerrLower, '--r')\n",
    "# plt.plot(xd, yerrUpper, '--r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jJVBP5g0Ffm3"
   },
   "outputs": [],
   "source": [
    "yd = [comm_dic2[k]['avg_sentiment'] for k,z in load_dic.items()]\n",
    "xd = [load_dic[k] for k,z in load_dic.items()]\n",
    "\n",
    "plt.figure(figsize=(10,10))\n",
    "\n",
    "# make the scatter plot\n",
    "plt.scatter(xd, yd, s=30, alpha=0.75, marker='o')\n",
    "\n",
    "# determine best fit line\n",
    "par = np.polyfit(xd, yd, 1, full=True)\n",
    "\n",
    "slope=par[0][0]\n",
    "intercept=par[0][1]\n",
    "xl = [min(xd), max(xd)]\n",
    "yl = [slope*xx + intercept  for xx in xl]\n",
    "\n",
    "# coefficient of determination, plot text\n",
    "variance = np.var(yd)\n",
    "residuals = np.var([(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)])\n",
    "Rsqr = np.round(1-residuals/variance, decimals=2)\n",
    "plt.text(.5*max(xd)+.1*min(xd),0.25*max(yd)+.1*min(yd),'$R^2 = %0.2f$'% Rsqr, fontsize=30)\n",
    "\n",
    "plt.ylabel(\"Weighted Sentiment\",size=20)\n",
    "plt.xlabel(\"Load\",size=20)\n",
    "plt.title('Weighted Sentiment vs. Load',size=25)\n",
    "\n",
    "# error bounds\n",
    "yerr = [abs(slope*xx + intercept - yy)  for xx,yy in zip(xd,yd)]\n",
    "par = np.polyfit(xd, yerr, 2, full=True)\n",
    "\n",
    "# yerrUpper = [(xx*slope+intercept)+(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "# yerrLower = [(xx*slope+intercept)-(par[0][0]*xx**2 + par[0][1]*xx + par[0][2]) for xx,yy in zip(xd,yd)]\n",
    "\n",
    "for k,z in load_dic.items():\n",
    "  y,x = comm_dic2[k]['avg_sentiment'],load_dic[k]\n",
    "  if (abs(y) > 0.08) and k!='French' or k=='Dutch':\n",
    "    plt.text(x+0.003,y+.0025, k, fontsize=10,fontweight='bold')\n",
    "  if (abs(y) > 0.08) and k=='French':\n",
    "    plt.text(x+0.003,y-.015, k, fontsize=10,fontweight='bold')\n",
    "\n",
    "plt.plot(xl, yl, '-r')\n",
    "plt.axhline(y=0, color='k', linestyle='-')\n",
    "# plt.plot(xd, yerrLower, '--r')\n",
    "# plt.plot(xd, yerrUpper, '--r')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Hqa33I1bSLUR"
   },
   "source": [
    "###10b. Density"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bnx_vI1ZSQH3"
   },
   "outputs": [],
   "source": [
    "widths = [e_dic_cl[i]['width'] for i in G_cl.edges]\n",
    "maxw = max(widths)\n",
    "n = len(widths)\n",
    "weighted_edge = round(sum(widths)/(n*(n-1)/2*maxw),5)\n",
    "print('The Density of G is: {}'.format(round(nx.density(G_cl),3)))\n",
    "print('The Weighted Density of G is: {}'.format(weighted_edge))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wJcU09kyUYbP"
   },
   "source": [
    "###10c. Degree Histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zO22K2RbYIXo"
   },
   "outputs": [],
   "source": [
    "def weight_attr(G_cl,maxw):\n",
    "    weight={}\n",
    "    for edge in G_cl.edges():\n",
    "        ed=edge[0]\n",
    "        de=edge[1]\n",
    "        G_cl[ed][de]['weight']=random.randrange(1,maxw)\n",
    "        weight[(ed,de)]=G_cl[ed][de]['weight']\n",
    "    return weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_tc-lfENbh9b"
   },
   "outputs": [],
   "source": [
    "from pylab import hist\n",
    "\n",
    "ds=dict(G_cl.degree(G_cl.nodes())).values()\n",
    "bins=7\n",
    "\n",
    "plt.figure(figsize=(20,7))\n",
    "plt.subplot(121)\n",
    "plt.xlabel(\"Degree\");\n",
    "hist(ds,bins=bins,color='g'); \n",
    "# histtype='step',\n",
    "plt.ylabel(\"Number of nodes\");\n",
    "ss='Degree Histogram of G' \n",
    "plt.title(ss,fontsize=20);\n",
    "plt.subplot(122)\n",
    "plt.xlabel(\"Degree\");\n",
    "hist(ds,bins=bins,color='g',log=True); \n",
    "# histtype='step',\n",
    "plt.ylabel(\"Number of nodes\");\n",
    "sss='Log Degree Histogram of G' \n",
    "plt.title(sss,fontsize=20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "X7_vkRbimj6H"
   },
   "outputs": [],
   "source": [
    "k = []\n",
    "Pk = []\n",
    "\n",
    "for node in list(G_cl.nodes()):\n",
    "    degree = G_cl.degree(nbunch=node)\n",
    "    try:\n",
    "        post = k.index(degree)\n",
    "    except ValueError as e:\n",
    "        k.append(degree)\n",
    "        Pk.append(1)\n",
    "    else:\n",
    "        Pk[post] += 1\n",
    "\n",
    "logk=[]\n",
    "logPk=[]\n",
    "# get a double log representation\n",
    "for i in range(len(k)):\n",
    "    if k[i]>0:\n",
    "        logk.append(math.log10(k[i]))\n",
    "        logPk.append(math.log10(Pk[i]))\n",
    "\n",
    "order = np.argsort(logk)\n",
    "logk_array = np.array(logk)[order]\n",
    "logPk_array = np.array(logPk)[order]\n",
    "plt.plot(logk_array, logPk_array, \".\")\n",
    "m, c = np.polyfit(logk_array, logPk_array, 1)\n",
    "print(\"m =\",m, \"c =\",c)\n",
    "plt.plot(logk_array, m*logk_array + c, \"-\");\n",
    "\n",
    "if m>-3 and m<-2:\n",
    "    print(\"The %s is scale-free\" %name)\n",
    "else:\n",
    "    print(\"The %s is not scale-free\" %name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5cJ6p-SzUmaE"
   },
   "outputs": [],
   "source": [
    "dsw=dict(G_cl.degree(G_cl.nodes()),weight='weight').values()\n",
    "bins=7\n",
    "\n",
    "plt.figure(figsize=(20,7))\n",
    "plt.subplot(121)\n",
    "plt.xlabel(\"Weighted Degree\");\n",
    "hist(dsw,bins=bins,color='g'); \n",
    "# histtype='step',\n",
    "plt.ylabel(\"Number of nodes\");\n",
    "ss='Weighted Degree Histogram of G' \n",
    "plt.title(ss,fontsize=20);\n",
    "plt.subplot(122)\n",
    "plt.xlabel(\"Weighted Degree\");\n",
    "hist(dsw,bins=bins,color='g',log=True); \n",
    "# histtype='step',\n",
    "plt.ylabel(\"Number of nodes\");\n",
    "sss='Log Weighted Degree Histogram of G' \n",
    "plt.title(sss,fontsize=20);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8CmDWVpomdhy"
   },
   "outputs": [],
   "source": [
    "k = []\n",
    "Pk = []\n",
    "\n",
    "for node in list(G_cl.nodes()):\n",
    "    degree = G_cl.degree(nbunch=node,weight='weight')\n",
    "    try:\n",
    "        post = k.index(degree)\n",
    "    except ValueError as e:\n",
    "        k.append(degree)\n",
    "        Pk.append(1)\n",
    "    else:\n",
    "        Pk[post] += 1\n",
    "\n",
    "logk=[]\n",
    "logPk=[]\n",
    "# get a double log representation\n",
    "for i in range(len(k)):\n",
    "    if k[i]>0:\n",
    "        logk.append(math.log10(k[i]))\n",
    "        logPk.append(math.log10(Pk[i]))\n",
    "\n",
    "order = np.argsort(logk)\n",
    "logk_array = np.array(logk)[order]\n",
    "logPk_array = np.array(logPk)[order]\n",
    "plt.plot(logk_array, logPk_array, \".\")\n",
    "m, c = np.polyfit(logk_array, logPk_array, 1)\n",
    "print(\"m =\",m, \"c =\",c)\n",
    "plt.plot(logk_array, m*logk_array + c, \"-\");\n",
    "\n",
    "if m>-3 and m<-2:\n",
    "    print(\"The wighted %s is scale-free\" %name)\n",
    "else:\n",
    "    print(\"The weighted %s is not scale-free\" %name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nQ8xgTXpbGJe"
   },
   "source": [
    "###10d. Connectivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wWDA4GKmbJw5"
   },
   "outputs": [],
   "source": [
    "if nx.is_connected(G_cl)==True:\n",
    "    print (\"This graph is a connected graph\")\n",
    "else:\n",
    "    print (\"This graph is a disconnected graph and it has\",nx.number_connected_components(sGw),\"connected components\" )   \n",
    "    giant = max(nx.connected_component_subgraphs(G_cl), key=len)\n",
    "    sGwlcc=sGw.subgraph(giant)\n",
    "    print (\"The largest connected component of this graph is a weighted graph with %i nodes and %i edges\" %(len(sGwlcc.nodes()),len(sGwlcc.edges())))\n",
    "    print (\"The density of the largest connected component of this graph is %.3f\" %nx.density(sGwlcc))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8PfUe7B_beQE"
   },
   "source": [
    "###10e. Transitivity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2aUSRuKWbttO"
   },
   "outputs": [],
   "source": [
    "print('The transitivity of G is: {}'.format(round(nx.transitivity(G_cl),3)))\n",
    "#Compute graph transitivity, the fraction of all possible triangles present in G.\n",
    "#Possible triangles are identified by the number of “triads” (two edges with a shared vertex)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dax8Eg-LcmZT"
   },
   "source": [
    "###10f. Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9KOhiHVDctGP"
   },
   "outputs": [],
   "source": [
    "Gt = nx.Graph()\n",
    "wd = []\n",
    "Gt.add_nodes_from(G_cl.nodes)\n",
    "for edge in G_cl.edges:\n",
    "  wd.append((edge[0],edge[1],e_dic_cl[edge]['width']))\n",
    "Gt.add_weighted_edges_from(wd)\n",
    "print('The weighted clustering coefficients are:')\n",
    "clusdic = nx.clustering(Gt,weight='weight')\n",
    "for z,v in clusdic.items():\n",
    "  clusdic[z] = round(v,3)\n",
    "clusdic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4c8ZEf074IY9"
   },
   "outputs": [],
   "source": [
    "print('The average weighted clustering is: {}'.format(round(nx.average_clustering(Gt, nodes=None, weight='weight', count_zeros=True),3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kK-Rl41j4-qs"
   },
   "source": [
    "###10g. Distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Kgd6zpZt5EDd"
   },
   "outputs": [],
   "source": [
    "print('The diamaeter of G is: {}'.format(nx.diameter(Gt)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EubgjDsW5dQc"
   },
   "outputs": [],
   "source": [
    "print('The average shortest path of G is: {}'.format(round(nx.average_shortest_path_length(Gt),2)))\n",
    "print('The weighted average shortest path of G is: {}'.format(round(nx.average_shortest_path_length(Gt, weight='weight'),2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Rvht3ZPi571t"
   },
   "source": [
    "###10g. Assortativity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7WvqRIaM6CQ0"
   },
   "outputs": [],
   "source": [
    "print('The weighted degree assortativity coefficient is: {}'.format(round(nx.degree_assortativity_coefficient(Gt, x='out', y='in', weight='weight', nodes=None),2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t1zmM9JTnMw5"
   },
   "outputs": [],
   "source": [
    "print('The attribute assortativity coefficient is: {}'.format(get_att_assort(Gt,n_dic_cl)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6aTlO3a5l0je"
   },
   "outputs": [],
   "source": [
    "def make_mapping(G,n_dic,att='tag'):\n",
    "  mapping = {}\n",
    "  i = 0\n",
    "  for node in G.nodes:\n",
    "    if n_dic[node][att] not in mapping.keys():\n",
    "      mapping[n_dic[node][att]] = i\n",
    "      i += 1\n",
    "  return mapping\n",
    "nx.set_node_attributes(Gt, {z:n_dic_cl[z]['tag'] for z,v in n_dic.items()}, 'tag')\n",
    "mapping = make_mapping(Gt,n_dic_cl,att='tag')\n",
    "mix_mat = nx.attribute_mixing_matrix(Gt, 'tag',mapping=mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8q42QfND0EUU"
   },
   "outputs": [],
   "source": [
    "from matplotlib import colors\n",
    "print('Mixing matrix of G by NLP tag:')\n",
    "df_mix = pd.DataFrame(data=mix_mat,index=mapping.keys(),columns=mapping.keys())*len(Gt.edges)*2\n",
    "def background_gradient(s, m, M, cmap='PuBu', low=0, high=0):\n",
    "    rng = M - m\n",
    "    norm = colors.Normalize(m - (rng * low),\n",
    "                            M + (rng * high))\n",
    "    normed = norm(s.values)\n",
    "    c = [colors.rgb2hex(x) for x in plt.cm.get_cmap(cmap)(normed)]\n",
    "    return ['background-color: %s' % color for color in c]\n",
    "\n",
    "df_mix.style.apply(background_gradient,\n",
    "               cmap='cool',\n",
    "               m=df_mix.min().min(),\n",
    "               M=df_mix.max().max(),\n",
    "               low=0,\n",
    "               high=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z8Fe6ju6df6s"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "CIS_Thesis_3_20_2020Rev5.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}