contingencyTableAnl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3a1591fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objects as go # for fancy interactive plot\n",
    "import seaborn as sns\n",
    "import requests\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import os # to check if directory exists and create it if it doesn't\n",
    "from datetime import datetime # to parse speech date\n",
    "from nrclex import NRCLex\n",
    "import spacy\n",
    "import en_core_web_md\n",
    "import csv\n",
    "import prince # for correspondence analysis\n",
    "import nltk\n",
    "from scipy.stats import power_divergence # for g-test (log likelihood ratio)\n",
    "from scipy.stats.contingency import association\n",
    "from scipy.stats import chi2_contingency\n",
    "from scipy.stats import chi2\n",
    "stopwords = pd.read_table('./Data/word_lists/kaggle_stopwords.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9621696f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cont_table = pd.read_csv('./Data/genData/ContingencyTable.csv', index_col=0)\n",
    "cont_table = pd.read_csv('./Data/genData/contingency_table_noSW_oba.csv', index_col=0)\n",
    "cont_table = cont_table.astype(int)\n",
    "ctab_gwb = pd.read_csv('./Data/genData/contingency_table_noSW_gwb.csv', index_col=0)\n",
    "ctab_gwb = ctab_gwb.astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75e0054c",
   "metadata": {},
   "source": [
    "<A HREF=\"https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.power_divergence.html\">SciPy power divergence for G-test or log-likelihood ratio</A> set lambda_='log-likelihood' for g-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cbb9f146",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/muddy/.local/lib/python3.10/site-packages/scipy/stats/_stats_py.py:7410: RuntimeWarning: invalid value encountered in divide\n",
      "  terms = 2.0 * special.xlogy(f_obs, f_obs / f_exp)\n"
     ]
    }
   ],
   "source": [
    "g_test = power_divergence(cont_table, lambda_='log-likelihood')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "005ce662",
   "metadata": {},
   "source": [
    "<A HREF=\"https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.contingency.association.html#scipy.stats.contingency.association\">SciPy association</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4376f259",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7716"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(g_test[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85259b9b",
   "metadata": {},
   "source": [
    "<A HREF=\"https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.chi2_contingency.html\">SciPy contingency table Chi2 test</A>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c5b6dae",
   "metadata": {},
   "source": [
    "from <A HREF=\"https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test\">Wikipedia</A> : In cases where the expected value, E, is found to be small (indicating a small underlying population probability, and/or a small number of observations), the normal approximation of the multinomial distribution can fail, and in such cases it is found to be more appropriate to use the G-test, a likelihood ratio-based test statistic"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02404770",
   "metadata": {},
   "source": [
    "Starting with <A HREF=\"https://aclanthology.org/J93-1003/\">Dunning 1993</A> the g-test (likelihood ratio) became popular for computational linguistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5dc2a2e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Problems with no stopwords version of Obama contingency table ..... arghhh\n",
    "res_gtest = chi2_contingency(cont_table, lambda_=\"log-likelihood\") # g-test\n",
    "print(\"Statistic:\",res_gtest[0],\"\\tp-value:\",res_gtest[1],\"\\tdf:\",res_gtest[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ae200cd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "col_sums = np.reshape(cont_table.sum(axis=0).to_numpy(), (-1,1))\n",
    "row_sums = np.reshape(cont_table.sum(axis=1).to_numpy(), (-1,1))\n",
    "total = row_sums.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "67457204",
   "metadata": {},
   "outputs": [],
   "source": [
    "expected = pd.DataFrame(np.matmul(row_sums, np.transpose(col_sums))/total, columns=cont_table.columns, index=cont_table.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74f07cfe",
   "metadata": {},
   "source": [
    "### Compare words which occur more than expected - Obama vs. Bush"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4dc3af01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "know         1.494954e-16\n",
       "world        1.099231e-16\n",
       "serve        9.893076e-17\n",
       "tonight      9.013692e-17\n",
       "insurance    8.683923e-17\n",
       "stand        8.354153e-17\n",
       "challenge    8.134307e-17\n",
       "human        8.134307e-17\n",
       "year         7.914461e-17\n",
       "care         7.694615e-17\n",
       "trade        7.035077e-17\n",
       "place        7.035077e-17\n",
       "home         7.035077e-17\n",
       "family       7.035077e-17\n",
       "effort       6.815230e-17\n",
       "like         6.595384e-17\n",
       "american     6.595384e-17\n",
       "high         6.265615e-17\n",
       "oil          6.155692e-17\n",
       "end          5.716000e-17\n",
       "dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Obama Words occuring more often than expected\n",
    "difference = cont_table.subtract(expected)\n",
    "difference.mean(axis=0).sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a4f00a76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "people       3.243760\n",
       "work         2.252611\n",
       "country      2.150493\n",
       "help         1.952263\n",
       "world        1.898200\n",
       "new          1.886186\n",
       "american     1.844138\n",
       "nation       1.766047\n",
       "year         1.711985\n",
       "time         1.561810\n",
       "great        1.459692\n",
       "come         1.429657\n",
       "child        1.411636\n",
       "need         1.381602\n",
       "terrorist    1.363581\n",
       "life         1.363581\n",
       "freedom      1.309518\n",
       "know         1.309518\n",
       "today        1.213407\n",
       "economy      1.159344\n",
       "dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_sums_gwb = np.reshape(ctab_gwb.sum(axis=0).to_numpy(), (-1,1))\n",
    "row_sums_gwb = np.reshape(ctab_gwb.sum(axis=1).to_numpy(), (-1,1))\n",
    "total_gwb = row_sums_gwb.sum()\n",
    "expected_gwb = pd.DataFrame(np.matmul(row_sums_gwb, np.transpose(col_sums_gwb))/total, \n",
    "                        columns=ctab_gwb.columns, index=ctab_gwb.index)\n",
    "difference_gwb = ctab_gwb.subtract(expected_gwb)\n",
    "# GWB Words occuring more often than expected\n",
    "difference_gwb.mean(axis=0).sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1066d2c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stem            1.332268e-16\n",
      "year            1.097162e-16\n",
      "world           7.836868e-17\n",
      "economic        7.053182e-17\n",
      "learn           6.791953e-17\n",
      "pass            6.138880e-17\n",
      "today           5.485808e-17\n",
      "terror          5.224579e-17\n",
      "come            4.963350e-17\n",
      "agency          4.571507e-17\n",
      "new             4.179663e-17\n",
      "future          4.049049e-17\n",
      "confidence      4.049049e-17\n",
      "achieve         4.049049e-17\n",
      "story           3.918434e-17\n",
      "long            3.918434e-17\n",
      "resolution      3.853127e-17\n",
      "income          3.657205e-17\n",
      "peace           3.657205e-17\n",
      "compensation    3.526591e-17\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "def top_20_diff(ctab):\n",
    "    col_sums = np.reshape(ctab.sum(axis=0).to_numpy(), (-1,1))\n",
    "    row_sums = np.reshape(ctab.sum(axis=1).to_numpy(), (-1,1))\n",
    "    total = row_sums.sum()\n",
    "    expected = pd.DataFrame(np.matmul(row_sums, np.transpose(col_sums))/total, \n",
    "                            columns=ctab.columns, index=ctab.index)\n",
    "    difference = ctab.subtract(expected)\n",
    "    # GWB Words occuring more often than expected\n",
    "    print(difference.mean(axis=0).sort_values(ascending=False).head(20))\n",
    "top_20_diff(ctab_gwb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2c58ac7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "know         1.494954e-16\n",
      "world        1.099231e-16\n",
      "serve        9.893076e-17\n",
      "tonight      9.013692e-17\n",
      "insurance    8.683923e-17\n",
      "stand        8.354153e-17\n",
      "challenge    8.134307e-17\n",
      "human        8.134307e-17\n",
      "year         7.914461e-17\n",
      "care         7.694615e-17\n",
      "trade        7.035077e-17\n",
      "place        7.035077e-17\n",
      "home         7.035077e-17\n",
      "family       7.035077e-17\n",
      "effort       6.815230e-17\n",
      "like         6.595384e-17\n",
      "american     6.595384e-17\n",
      "high         6.265615e-17\n",
      "oil          6.155692e-17\n",
      "end          5.716000e-17\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "top_20_diff(cont_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "8a0e73ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "people           18.761196\n",
      "ideal            15.006051\n",
      "nation           13.126060\n",
      "international    12.151876\n",
      "world            11.169223\n",
      "young             8.033683\n",
      "individual        7.586729\n",
      "human             7.058693\n",
      "meet              6.785196\n",
      "russian           6.751513\n",
      "believe           6.527229\n",
      "history           6.406414\n",
      "voice             5.928399\n",
      "law               5.899960\n",
      "free              5.810206\n",
      "conflict          5.285599\n",
      "force             5.220049\n",
      "stand             5.117387\n",
      "come              4.862042\n",
      "ethnic            4.860226\n",
      "Name: 2014-03-26-NATOBelgium, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "def speech_diff(ctab, date):\n",
    "    col_sums = np.reshape(ctab.sum(axis=0).to_numpy(), (-1,1))\n",
    "    row_sums = np.reshape(ctab.sum(axis=1).to_numpy(), (-1,1))\n",
    "    total = row_sums.sum()\n",
    "    expected = pd.DataFrame(np.matmul(row_sums, np.transpose(col_sums))/total, \n",
    "                            columns=ctab.columns, index=ctab.index)\n",
    "    diff = ctab.subtract(expected)\n",
    "    row = diff.iloc[diff.index.str.startswith(date).argmax()]\n",
    "    sorted_row_desc = row.sort_values(ascending=False)\n",
    "    print(sorted_row_desc[:20])\n",
    "    \n",
    "speech_diff(cont_table, '2014-03-26')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a71b10d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "### From obama_vs_gwb.ipynb PCA of obama speechs, far left on plot = '2016-05-06', upper right = '2016-03-23'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "76dd4e2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "water         36.421945\n",
      "kid           19.642423\n",
      "lead          18.532012\n",
      "city          16.421945\n",
      "sure          15.779177\n",
      "say           15.406253\n",
      "government    13.946283\n",
      "mindset       13.695797\n",
      "community     13.322872\n",
      "child         13.296186\n",
      "happen        10.425610\n",
      "start         10.425610\n",
      "feel           9.946627\n",
      "tell           9.539686\n",
      "hear           9.193792\n",
      "invest         8.908601\n",
      "filter         8.828886\n",
      "people         7.895887\n",
      "problem        7.718474\n",
      "raise          7.231818\n",
      "Name: 2016-05-06-FlintWaterCrisis, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "speech_diff(cont_table, '2016-05-06')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "127af55b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "people         3.176832\n",
      "naval          2.997034\n",
      "vessel         1.997034\n",
      "fly            1.994067\n",
      "station        1.991842\n",
      "flag           1.985910\n",
      "staff          1.982943\n",
      "facility       1.971819\n",
      "half           1.945864\n",
      "military       1.885795\n",
      "fortieth       0.999258\n",
      "consular       0.999258\n",
      "legation       0.999258\n",
      "hereunto       0.999258\n",
      "perpetrate     0.998517\n",
      "sunset         0.998517\n",
      "vest           0.998517\n",
      "sixteen        0.997775\n",
      "length         0.997775\n",
      "perpetrator    0.996292\n",
      "Name: 2016-03-23-BrusselsTerrorism, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "speech_diff(cont_table, '2016-03-23')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac4831cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# And speeches on opposite sides from GWB's PCA plot: '2008-09-04' and '2003-02-01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "5087b3a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fight         30.457572\n",
      "country       18.680385\n",
      "know          17.498111\n",
      "work          13.095375\n",
      "let            9.694374\n",
      "stand          9.180231\n",
      "friend         9.143776\n",
      "lose           8.210559\n",
      "government     8.113449\n",
      "job            7.977842\n",
      "change         7.455530\n",
      "love           7.281426\n",
      "think          6.313796\n",
      "bad            6.139692\n",
      "tough          6.002042\n",
      "party          5.728786\n",
      "cause          5.556724\n",
      "believe        5.455530\n",
      "bless          5.451445\n",
      "big            5.208516\n",
      "Name: 2008-09-04-presidential-nominat, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "speech_diff(ctab_gwb, '2008-09-04')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "3cb68377",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "naval         2.995696\n",
      "vessel        1.997848\n",
      "station       1.995696\n",
      "staff         1.991391\n",
      "flag          1.991391\n",
      "facility      1.989239\n",
      "fly           1.986011\n",
      "half          1.972022\n",
      "military      1.895621\n",
      "hereunto      0.998924\n",
      "legation      0.998924\n",
      "consular      0.998924\n",
      "possession    0.998924\n",
      "aboard        0.998924\n",
      "vest          0.997848\n",
      "embassy       0.997848\n",
      "seventh       0.997848\n",
      "length        0.997848\n",
      "building      0.991391\n",
      "post          0.989239\n",
      "Name: 2003-02-01-7646-honoring-memory, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "speech_diff(ctab_gwb, '2003-02-01')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "9c2ff047",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nation        9.801565\n",
      "generation    6.154402\n",
      "common        4.923401\n",
      "spirit        4.364294\n",
      "new           4.171286\n",
      "man           4.098309\n",
      "seek          4.056694\n",
      "meet          3.877562\n",
      "understand    3.503015\n",
      "father        3.446321\n",
      "carry         3.436067\n",
      "let           3.401083\n",
      "endure        3.302774\n",
      "remain        3.169481\n",
      "word          3.056694\n",
      "far           2.974668\n",
      "long          2.862483\n",
      "moment        2.861881\n",
      "woman         2.682749\n",
      "crisis        2.656815\n",
      "Name: 2009-01-20-Inauguration, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Obama first inaugural address\n",
    "speech_diff(cont_table, '2009-01-20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "40d552f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "story         9.626142\n",
      "citizen       7.930429\n",
      "nation        7.074491\n",
      "common        4.639494\n",
      "ideal         4.599438\n",
      "courage       4.412509\n",
      "country       4.219958\n",
      "commitment    4.198876\n",
      "promise       3.985242\n",
      "civility      3.919888\n",
      "deep          3.692902\n",
      "time          3.528461\n",
      "character     3.492621\n",
      "purpose       3.479269\n",
      "principle     3.465917\n",
      "public        3.345748\n",
      "duty          3.198876\n",
      "community     3.011947\n",
      "life          2.969079\n",
      "grand         2.946592\n",
      "Name: 2001-01-20-inaugural-52, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# GWB first inaugural address\n",
    "speech_diff(ctab_gwb, '2001-01-20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "27461d2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "job         26.703030\n",
       "tax         20.635265\n",
       "right       16.577658\n",
       "company     14.161356\n",
       "business    13.460690\n",
       "plan        13.392313\n",
       "cut         13.165558\n",
       "pass        11.505691\n",
       "build        9.805026\n",
       "away         9.445106\n",
       "worker       8.614866\n",
       "economy      8.298722\n",
       "hire         8.044388\n",
       "pay          7.902207\n",
       "ask          7.100771\n",
       "idea         7.056383\n",
       "work         6.908773\n",
       "small        6.748643\n",
       "proposal     6.384521\n",
       "meet         5.647260\n",
       "Name: 2011-09-09-SpeechJobsAct, dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sort words from one speech by difference between observed and expected\n",
    "date = '2011-09-09'\n",
    "# Words occuring more often than expected - over represented\n",
    "difference[difference.index.str.startswith(date)].iloc[0].sort_values(ascending=False)[0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "14dd507a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "military   -2.494309\n",
       "love       -2.542900\n",
       "progress   -2.542900\n",
       "ago        -2.623884\n",
       "student    -2.623884\n",
       "end        -2.676676\n",
       "effort     -2.838644\n",
       "child      -2.859044\n",
       "force      -2.899229\n",
       "nuclear    -2.915426\n",
       "day        -3.037209\n",
       "stand      -3.049203\n",
       "war        -3.146384\n",
       "energy     -3.417527\n",
       "come       -3.487130\n",
       "think      -3.502714\n",
       "change     -3.928033\n",
       "like       -4.547714\n",
       "say        -5.320959\n",
       "today      -6.320959\n",
       "Name: 2011-09-09-SpeechJobsAct, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Words occuring less often than expected\n",
    "difference[difference.index.str.startswith(date)].iloc[0].sort_values(ascending=False)[-20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52fc18ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f4c61b4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOAD CONTINGENCY TABLE\n",
    "# Read in saved contingency table\n",
    "ctab = pd.read_csv('./Data/genData/obamaContingencyTable_noSW.csv',index_col=0) \n",
    "# Shorten row index names to date only\n",
    "ctab = ctab.rename(index = lambda x: x[0:10])\n",
    "ca = prince.CA(n_components=2,\n",
    "               n_iter=3,copy=True,\n",
    "               check_input=True,\n",
    "               engine='sklearn',\n",
    "               random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4e91509",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d31c30b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e12b4df5",
   "metadata": {},
   "source": [
    "### Ok, let's cram this into one cell. Enter data of interest at top... out pops a nice CA plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbe87e3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2011-07-26 debt ceiling\n",
    "# 2011-05-02 Osama bin Laden killed\n",
    "# 2011-09-09 Speech Jobs Act\n",
    "# 2011-06-23 Afghanistan Pullout\n",
    "# 2011-05-23 Missouri Tornado\n",
    "\n",
    "# Sort words from one speech by difference between observed and expected\n",
    "date = '2011-05-02'\n",
    "# label location adjustment\n",
    "x=0.1\n",
    "y=-0.1\n",
    "#make word list from difference between observed and expected\n",
    "over_rep = difference[difference.index.str.startswith(date)].iloc[0].sort_values(ascending=False)[0:75]\n",
    "under_rep = difference[difference.index.str.startswith(date)].iloc[0].sort_values(ascending=False)[-75:]\n",
    "over_rep = pd.DataFrame(over_rep)\n",
    "under_rep = pd.DataFrame(under_rep)\n",
    "\n",
    "word_list = pd.concat([over_rep, under_rep])\n",
    "word_list.columns = ['difference']\n",
    "word_list['word']=word_list.index\n",
    "word_list.drop(word_list.loc[word_list['word']=='â'].index, inplace=True) # get rid of that pesky goofy a\n",
    "# Make a short contingency list using topic words\n",
    "ctabshort = ctab.loc[ : , [col for col in ctab.columns if col in list(word_list['word'])]]\n",
    "ctabshort.dropna(inplace=True)\n",
    "# Keep it to the most common 150 words\n",
    "if np.shape(ctabshort)[1] > 150:\n",
    "    ctabshort=ctabshort.iloc[:, : 150]\n",
    "    \n",
    "ctabshort.columns.rename('words', inplace=True)\n",
    "ctabshort.index.rename('speeches', inplace=True)\n",
    "# Fit Correspondence Analysis with speeches and specified word list\n",
    "ca = ca.fit(ctabshort)\n",
    "\n",
    "date1 = '2011-05-02'     # date1 - red X   --- uncomment ax.plot line 46 to highlight two points\n",
    "date2 = date             # date2 - green pentagon \n",
    "df1=ca.row_coordinates(ctabshort)\n",
    "df1.columns=['Comp1', 'Comp2']\n",
    "df2=ca.column_coordinates(ctabshort)\n",
    "df2.columns=['Comp1', 'Comp2']\n",
    "\n",
    "# df1 for speeches\n",
    "ax = df1.plot(x='Comp1', y='Comp2', figsize=(10, 10), kind='scatter', color='blue')\n",
    "#Label speech points\n",
    "#for i, txt in enumerate(df1.index.values.tolist()):\n",
    "#    ax.annotate(txt, (df1['Comp1'][i], df1['Comp2'][i]))\n",
    "\n",
    "# df2 for words\n",
    "df2.plot(ax=ax, x='Comp1', y='Comp2', kind='scatter', color='orange')\n",
    "for i, txt in enumerate(df2.index.values.tolist()):\n",
    "    ax.annotate(txt, (df2['Comp1'][i], df2['Comp2'][i]))\n",
    "\n",
    "#ax.plot(df1.loc[[date1]]['Comp1'], df1.loc[[date1]]['Comp2'], marker='X', color='red', markersize=20)\n",
    "#ax.annotate(date1, (df1['Comp1'][date1]+0.1, df1['Comp2'][date1]), color='darkred')\n",
    "ax.plot(df1.loc[[date2]]['Comp1'], df1.loc[[date2]]['Comp2'], marker='p', color='green', markersize=20)\n",
    "ax.annotate(date2, (df1['Comp1'][date2]+x, df1['Comp2'][date2]+y), color='darkgreen')\n",
    "\n",
    "#plt.xlim([-0.25, 5.0])\n",
    "#plt.ylim([-1.2, 1.0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df9d7c5a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}