bertTopics_win.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa44ebe5",
   "metadata": {},
   "source": [
    "<A HREF=\"https://theaidigest.in/topic-modeling-using-roberta-and-transformers/\">Topic modeling using Roberta and transformers</A><BR>\n",
    "    <A HREF=\"https://maartengr.github.io/BERTopic/index.html\">BERTopic documentation</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9d5b68d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install bertopic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d4f0c35e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\graph_objs\\__init__.py:288: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if LooseVersion(ipywidgets.__version__) >= LooseVersion(\"7.0.0\"):\n",
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\graph_objs\\__init__.py:288: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if LooseVersion(ipywidgets.__version__) >= LooseVersion(\"7.0.0\"):\n",
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\graph_objs\\__init__.py:288: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if LooseVersion(ipywidgets.__version__) >= LooseVersion(\"7.0.0\"):\n",
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\graph_objs\\__init__.py:288: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n",
      "  if LooseVersion(ipywidgets.__version__) >= LooseVersion(\"7.0.0\"):\n",
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\bertopic\\_utils.py:4: DeprecationWarning: Please use `csr_matrix` from the `scipy.sparse` namespace, the `scipy.sparse.csr` namespace is deprecated.\n",
      "  from scipy.sparse.csr import csr_matrix\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import spacy\n",
    "import en_core_web_md\n",
    "import csv\n",
    "import regex as re\n",
    "import os\n",
    "import spacy\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "from gensim.models import LdaModel\n",
    "from gensim.models import LdaMulticore\n",
    "from gensim.models import CoherenceModel\n",
    "from gensim import corpora\n",
    "import pprint\n",
    "\n",
    "import pyLDAvis.gensim_models\n",
    "pyLDAvis.enable_notebook() #Notebook visualisation enabled\n",
    "\n",
    "from bertopic import BERTopic # Much faster with GPU than on CPU\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction import text\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "# to add words to sklearn's countvectorizer stopword list\n",
    "from sklearn.feature_extraction import text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b1349ca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load up the files\n",
    "#path = './Data/DataUCSB/' # Smaller UCSB dataset\n",
    "path = './Data/amrhet/' # larger American Rhetoric dataset\n",
    "list_of_files = []\n",
    "\n",
    "for root, dirs, files in os.walk(path):\n",
    "    for file in files:\n",
    "        if file.endswith('.txt'):\n",
    "            list_of_files.append(os.path.join(root,file))\n",
    "\n",
    "filepath = './Data/DataUCSB/address-before-joint-session-the-congress-the-state-the-union-16.csv'\n",
    "speeches = []\n",
    "for file in list_of_files:\n",
    "    with open(file, encoding='utf-8') as f:\n",
    "        text = f.read()\n",
    "    f.close()\n",
    "    speeches.append(text)\n",
    "\n",
    "#clean out xa0 space characters\n",
    "[speech.replace(u'\\xa0', '') for speech in speeches]; # ; supresses output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09d1aef8",
   "metadata": {},
   "source": [
    "<A HREF=\"https://datagy.io/python-flatten-list-of-lists\">Flatten list of lists</A><BR>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d0c24cd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "listlist = [speech.split('.') for speech in speeches]\n",
    "\n",
    "# Flatten list template: [item for sublist in list_of_lists for item in sublist]\n",
    "# Here item is sentence, sublist is speech, list_of_lists is listlist\n",
    "sentences = [sentence.lstrip().replace(u'\\xa0', '') for speech in listlist for sentence in speech if len(sentence) > 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a24a39",
   "metadata": {},
   "source": [
    "<A HREF=\"https://www.sbert.net/#\">SBERT SentenceTransformers</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "59fe3ba3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare embeddings - this takes about 5-10 minutes and gets the GPU warmed up!\n",
    "# all-mpnet-base-v2 is about 400MB\n",
    "# Maybe try model_id = \"sentence-transformers/all-MiniLM-L6-v2\"\n",
    "sentence_model = SentenceTransformer(\"all-mpnet-base-v2\")\n",
    "embeddings = sentence_model.encode(sentences)\n",
    "\n",
    "# Sentence Embeddings done!\n",
    "#  now skip a few cells to where there is a nice big comment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "6b365731",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'When Richard came to the Oval Office, we talked about what those automatic budget cuts -- sequestration -- could mean for the VA'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[122]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "ba157638",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.46503104e-03,  3.14406492e-02, -5.08073671e-03, -3.96114737e-02,\n",
       "       -1.16334185e-02, -2.96727717e-02, -5.42856678e-02,  8.92422348e-03,\n",
       "       -7.73201063e-02, -4.46461141e-02,  5.16621023e-02,  2.76710652e-02,\n",
       "        5.11454600e-05,  1.42952045e-02, -2.27853954e-02,  1.93038199e-03,\n",
       "        6.41904548e-02, -3.04587297e-02, -1.52980909e-02, -2.19308771e-02,\n",
       "       -5.57330204e-03,  1.19999601e-02,  2.57524243e-03,  2.12978777e-02,\n",
       "        2.11541876e-02, -3.59955477e-03,  4.11274144e-03,  9.29337740e-03,\n",
       "       -6.88223122e-03, -5.48311509e-03,  7.84348976e-03,  1.76475253e-02,\n",
       "        3.47615480e-02, -1.13796256e-01,  1.93099117e-06, -1.03333062e-02,\n",
       "        1.30778717e-04,  5.71332872e-03, -1.01478718e-01, -3.00557483e-02,\n",
       "        2.07122006e-02, -6.32899627e-02,  7.57661462e-03, -1.33303087e-02,\n",
       "        2.91231135e-03, -4.23367228e-03, -1.11689391e-02, -3.08362693e-02,\n",
       "        1.57363303e-02, -4.59545897e-03, -8.12980230e-04,  1.75165571e-02,\n",
       "       -5.43145090e-02, -4.22631577e-02, -3.42215523e-02,  2.48678513e-02,\n",
       "        4.04039584e-02, -3.93847302e-02, -6.80288449e-02, -7.84300268e-02,\n",
       "       -7.03624391e-04,  2.45442390e-02,  1.44061148e-02,  5.83580509e-03,\n",
       "       -1.31648332e-02,  3.87291498e-02, -5.24043404e-02,  7.09909201e-02,\n",
       "       -1.10561503e-02, -4.71465886e-02,  9.97185707e-02,  5.80511242e-03,\n",
       "        3.72762941e-02,  5.81052043e-02,  3.33043151e-02,  7.37798680e-03,\n",
       "       -2.30598748e-02, -2.68941857e-02, -4.03241217e-02, -3.65353264e-02,\n",
       "       -3.88550712e-03,  3.25102247e-02,  2.09603761e-03,  1.14002386e-02,\n",
       "        5.61211966e-02, -4.59270831e-03,  5.33357356e-03, -1.16817690e-02,\n",
       "       -4.52994332e-02, -1.26636066e-02, -4.96262796e-02,  5.29369060e-03,\n",
       "        1.01762312e-02,  5.09188464e-03,  1.75222717e-02,  1.02426168e-02,\n",
       "       -6.61783665e-02,  3.36490460e-02,  3.52189243e-02,  3.52230459e-03,\n",
       "        6.72175689e-03, -9.96268541e-03,  3.26537900e-02, -7.73820386e-04,\n",
       "        4.42204028e-02,  4.41237986e-02, -4.90301773e-02, -3.12898979e-02,\n",
       "       -2.35886071e-02,  1.52999535e-02, -4.39851247e-02,  1.50185553e-02,\n",
       "       -4.45779823e-02,  5.97101729e-03, -3.69173847e-02, -5.18553145e-02,\n",
       "       -6.49301102e-03, -1.53903314e-03, -3.39340121e-02,  3.76633964e-02,\n",
       "       -7.55446078e-03, -2.14727670e-02, -7.44670108e-02, -3.85479257e-02,\n",
       "       -1.55762210e-02,  6.88325567e-03,  3.99326496e-02,  2.36469544e-02,\n",
       "       -2.36990470e-02,  6.18989915e-02,  3.16055422e-03, -1.94642767e-02,\n",
       "       -2.19213516e-02, -2.36271471e-02, -8.54569860e-03,  4.24493328e-02,\n",
       "        1.54557591e-02,  1.19334105e-02,  3.07926852e-02, -5.76290600e-02,\n",
       "        5.85991540e-04,  5.29146045e-02, -3.61571065e-03, -5.65787684e-03,\n",
       "       -1.11385705e-02, -4.88051735e-02,  4.88159917e-02,  4.72382084e-02,\n",
       "       -1.03804749e-02, -3.72125395e-02, -2.50030998e-02,  3.78609560e-02,\n",
       "        6.22291490e-03,  8.93979520e-03,  7.25911872e-04,  2.86938418e-02,\n",
       "        5.63529739e-03, -1.15589872e-02,  1.17268711e-02, -6.86662330e-04,\n",
       "       -5.15124528e-04,  1.54011799e-02, -8.54522288e-02, -1.14439251e-02,\n",
       "       -8.40158854e-03,  2.16852427e-02, -1.54555002e-02,  8.21233448e-03,\n",
       "       -6.21621199e-02, -5.00594871e-03,  9.77085717e-03,  1.98155139e-02,\n",
       "       -2.43076049e-02,  1.67064816e-02,  2.37186961e-02,  2.57511195e-02,\n",
       "       -1.03332177e-02,  7.15846778e-04,  5.96474344e-03, -1.36576397e-02,\n",
       "       -3.62285599e-02, -1.07729323e-02, -1.28185889e-02, -4.90752719e-02,\n",
       "       -1.47828348e-02,  7.41016213e-03,  5.94116412e-02, -4.06283699e-02,\n",
       "       -2.61618402e-02, -2.04625260e-02, -4.02848050e-03, -2.73261648e-02,\n",
       "        3.51953097e-02, -2.09332723e-02,  2.58583706e-02,  7.91364536e-03,\n",
       "       -5.15460782e-03,  5.24225645e-02, -3.72991338e-03,  1.06871417e-02,\n",
       "        8.66005023e-04, -7.15834498e-02, -6.18511289e-02,  4.37272489e-02,\n",
       "        3.68044674e-02, -5.48511837e-03, -5.06253242e-02, -3.35691012e-02,\n",
       "        5.06754369e-02, -1.89784486e-02,  8.16445872e-02,  4.07146923e-02,\n",
       "       -3.61673422e-02,  1.10638957e-03, -7.27468729e-03,  5.42926509e-03,\n",
       "        4.55184579e-02,  5.84321916e-02, -2.47350819e-02,  2.03910768e-02,\n",
       "       -4.58210930e-02,  5.41993557e-03,  1.36235906e-02,  2.45018415e-02,\n",
       "        3.22843976e-02,  5.51949143e-02,  3.53321284e-02,  2.08413135e-02,\n",
       "       -3.07977758e-03, -1.10429665e-02,  1.69522762e-02,  2.80499347e-02,\n",
       "       -4.21185009e-02, -5.40687554e-02, -3.47719737e-03, -4.05456014e-02,\n",
       "        3.18038166e-02,  4.05333191e-02, -1.55335469e-02, -2.30252300e-03,\n",
       "        4.62935679e-02, -7.17044063e-03, -1.31749816e-03, -2.50644032e-02,\n",
       "        6.83847666e-02, -6.74475506e-02,  1.12847677e-02,  1.53912837e-02,\n",
       "        1.92393661e-02,  4.22517397e-02,  1.48184709e-02,  7.60875456e-03,\n",
       "       -2.18140278e-02,  1.29443789e-02, -4.79205651e-03,  8.05501193e-02,\n",
       "        5.19206673e-02, -2.78593972e-03, -3.33343893e-02, -5.65230437e-02,\n",
       "        1.14249559e-02, -1.98810268e-02, -4.29350659e-02,  2.32471935e-02,\n",
       "       -3.71397212e-02, -4.50576693e-02,  3.88461426e-02,  2.68090963e-02,\n",
       "       -7.71041214e-03,  2.86739692e-02,  9.41154640e-03,  1.99706499e-02,\n",
       "       -3.59907709e-02, -4.96741245e-03,  2.02500112e-02,  9.26422551e-02,\n",
       "       -1.44447144e-02, -1.38492696e-03,  1.44304169e-04, -2.05336660e-02,\n",
       "        9.88448039e-03,  1.84047800e-02, -2.11643092e-02, -1.69259263e-03,\n",
       "       -2.23844927e-02,  4.09230124e-03,  5.49715534e-02, -1.02485241e-02,\n",
       "       -3.58454213e-02,  5.93809113e-02, -6.08525518e-03, -2.07388643e-02,\n",
       "       -1.86045785e-02, -8.29076171e-02,  1.56132132e-03,  1.69595648e-02,\n",
       "       -3.79539318e-02,  5.99422269e-02,  1.37629081e-02,  1.06260762e-01,\n",
       "        6.01092353e-02, -3.01098041e-02,  4.03650366e-02,  5.77421859e-03,\n",
       "       -2.09689289e-02,  2.17161383e-02,  2.36262064e-02,  4.45483960e-02,\n",
       "        1.99656915e-02, -1.66153386e-02,  2.61017401e-03,  3.69741656e-02,\n",
       "       -1.58575140e-02, -2.13014800e-03, -1.91673613e-03, -1.61617193e-02,\n",
       "        4.41361368e-02, -2.64793467e-02, -3.01949922e-02,  6.11040443e-02,\n",
       "        1.70353451e-04,  7.41996709e-03, -6.51552007e-02,  2.33803429e-02,\n",
       "       -2.49799136e-02, -1.37647437e-02, -3.07493415e-02, -3.57120000e-02,\n",
       "        4.04769406e-02, -1.92478541e-02, -1.88750010e-02,  3.11925486e-02,\n",
       "        5.72157372e-03, -3.55058461e-02,  1.05570843e-02,  4.58811596e-02,\n",
       "       -3.16211805e-02, -2.80709076e-03, -3.78418937e-02,  9.11985710e-03,\n",
       "       -1.92445163e-02, -1.65870339e-02, -1.80645566e-02,  5.13460785e-02,\n",
       "       -4.06244360e-02,  3.12207080e-02,  7.55054783e-03, -3.14651877e-02,\n",
       "       -7.45584071e-02, -1.63291264e-02, -2.63394266e-02,  2.64412519e-02,\n",
       "       -2.29504593e-02, -3.35444957e-02, -6.63692690e-03, -6.72695339e-02,\n",
       "        3.68903391e-02, -6.30980134e-02, -7.29647204e-02, -7.02218665e-03,\n",
       "        6.18769266e-02,  1.02552876e-03,  2.28003971e-02,  4.41192184e-03,\n",
       "       -1.13982651e-02,  3.01579703e-02,  3.04110628e-02,  3.23549844e-02,\n",
       "        8.13177973e-03,  3.45588550e-02, -2.38942653e-02, -3.31644081e-02,\n",
       "       -6.01552129e-02,  1.21321632e-02, -1.41564319e-02,  2.28819754e-02,\n",
       "        2.97880992e-02, -7.09322616e-02,  2.76900288e-02,  8.79904814e-03,\n",
       "        1.70931295e-02, -3.50320786e-02, -2.63354518e-02, -8.22560675e-03,\n",
       "       -4.71926220e-02, -9.75139719e-03,  1.34300888e-02, -4.01253104e-02,\n",
       "       -3.46391127e-02,  1.83778480e-02, -4.50279675e-02, -9.49774589e-03,\n",
       "        8.46746117e-02,  2.68068649e-02,  3.56319472e-02, -2.32824553e-02,\n",
       "       -1.12776384e-01, -2.23502312e-02, -1.54912537e-02, -2.27311184e-03,\n",
       "        3.69029157e-02, -4.29396564e-03,  9.35010239e-03, -9.92819015e-03,\n",
       "        2.71362979e-02, -3.19327638e-02,  3.37878950e-02,  2.19082460e-02,\n",
       "        9.31626186e-03,  3.40616666e-02,  4.35550064e-02,  5.41839898e-02,\n",
       "       -4.05226462e-02,  5.62149053e-03,  9.86930057e-02, -8.95777531e-03,\n",
       "        9.03918408e-03,  3.68722230e-02,  2.27305721e-02,  1.56749301e-02,\n",
       "       -4.33050059e-02,  5.30234491e-03, -5.40098399e-02, -4.67849150e-02,\n",
       "       -9.70205758e-03,  9.42914560e-03,  8.57844651e-02,  1.19329644e-02,\n",
       "        1.65479686e-02, -3.27603966e-02,  2.82578524e-02,  1.95741896e-02,\n",
       "        3.27592231e-02,  8.22005346e-02, -1.31154507e-02, -4.39610779e-02,\n",
       "        5.34969382e-02,  3.17440033e-02, -6.26177341e-02, -4.15577963e-02,\n",
       "        4.67555895e-02,  1.11746732e-02,  2.04700492e-02, -4.53741429e-03,\n",
       "        3.31363268e-02, -2.90541966e-02,  7.89070576e-02,  1.76071208e-02,\n",
       "       -1.48778250e-02, -5.12355287e-03, -1.97240487e-02, -5.61314402e-03,\n",
       "       -3.28971446e-02, -7.34250620e-02,  2.61160024e-02, -5.16785011e-02,\n",
       "       -3.13224234e-02,  8.30353517e-03, -2.03145351e-02,  3.30533050e-02,\n",
       "        4.79719006e-02,  3.83833796e-02,  4.01056930e-03,  3.09550501e-02,\n",
       "        1.99413337e-02,  1.86480079e-02,  3.82317826e-02,  2.16289125e-02,\n",
       "       -7.11442232e-02,  2.90927738e-02,  1.04661852e-01, -1.51762145e-03,\n",
       "        1.36153074e-02, -6.23239530e-03, -3.36285494e-02, -4.59118076e-02,\n",
       "       -1.52014634e-02,  5.49978984e-04, -3.54352742e-02, -2.79263537e-02,\n",
       "       -1.41676255e-02, -1.08149219e-02, -1.17137115e-02, -5.15045784e-02,\n",
       "       -2.39453409e-02, -1.29775573e-02, -1.11348275e-02, -1.14632947e-02,\n",
       "        1.21860474e-03,  2.13977937e-02, -5.24254404e-02, -2.24317861e-04,\n",
       "        3.31172384e-02,  7.52989296e-03,  2.46906932e-02,  3.35806385e-02,\n",
       "        9.20480583e-03,  5.28639406e-02,  3.79530974e-02,  4.64706607e-02,\n",
       "        4.47245650e-02,  2.12669298e-02,  4.23412398e-02,  4.97203181e-03,\n",
       "       -2.25661229e-02,  3.09959389e-02,  1.71275344e-02,  7.06202090e-02,\n",
       "        1.06725469e-02,  1.07053313e-02, -5.06771505e-02,  2.29949281e-02,\n",
       "       -5.22776283e-02, -8.88882205e-02,  9.12371106e-05, -2.64822524e-02,\n",
       "       -5.69751188e-02,  4.32858542e-02, -7.61156902e-03, -4.29786965e-02,\n",
       "        2.98879799e-02, -5.07956790e-03,  2.60863546e-02,  2.99286493e-03,\n",
       "        3.29666436e-02,  9.42324847e-02, -2.00904924e-02, -1.35806145e-03,\n",
       "       -5.11075966e-02,  3.68803367e-03, -2.45196256e-03,  8.20555463e-02,\n",
       "       -4.80081849e-02,  7.37287756e-03,  4.07598391e-02, -1.23141296e-01,\n",
       "        3.68363177e-03,  1.25348885e-02,  2.56115440e-02, -4.91286628e-03,\n",
       "        1.04367000e-03,  1.15351370e-02,  7.18431026e-02, -1.05741601e-02,\n",
       "        8.32146872e-03,  2.40316670e-02,  9.42874048e-03, -4.71056066e-02,\n",
       "        2.60825027e-02, -9.34127776e-04,  2.31073629e-02,  2.28988919e-02,\n",
       "       -8.05352405e-02,  6.67750428e-04, -1.75710656e-02, -5.70518469e-33,\n",
       "       -1.02370465e-02, -3.89745459e-02,  4.17319685e-03, -1.10280737e-02,\n",
       "       -2.79141832e-02,  1.04718678e-01,  2.04673503e-02,  1.45330653e-02,\n",
       "       -1.00721000e-02, -2.55108662e-02,  1.08144237e-02, -5.92440774e-04,\n",
       "        2.21422669e-02,  8.97807907e-03,  2.17979886e-02, -1.62334815e-02,\n",
       "        1.74019691e-02,  2.15735398e-02,  6.95094792e-03, -6.87020645e-03,\n",
       "       -3.19811702e-02,  3.91653888e-02,  3.97088714e-02,  8.65543168e-03,\n",
       "        1.23350015e-02,  8.48569050e-02,  4.02384847e-02, -3.53777520e-02,\n",
       "        4.06794846e-02, -5.65045048e-03, -1.57804620e-02, -5.70229366e-02,\n",
       "       -3.28307301e-02, -9.10282359e-02,  5.70858717e-02,  7.36158639e-02,\n",
       "        1.76405869e-02,  2.24346202e-02,  5.03568947e-02, -5.69488518e-02,\n",
       "       -2.06719190e-02, -8.97702947e-03, -5.95961139e-02,  7.23892003e-02,\n",
       "        1.70679800e-02, -1.96772013e-02, -1.85215008e-02, -1.01346793e-02,\n",
       "        3.90943233e-03,  3.60599533e-02,  5.61629748e-03,  4.53680120e-02,\n",
       "       -1.88004188e-02, -8.20736308e-03, -1.27083659e-02,  4.42946069e-02,\n",
       "       -2.19981726e-02,  5.65471798e-02,  3.06314938e-02,  4.86187264e-02,\n",
       "       -9.70128775e-02,  2.49006301e-02,  3.36003564e-02, -3.88208516e-02,\n",
       "       -1.56655144e-02,  1.24683995e-02, -3.49254645e-02,  3.18161328e-04,\n",
       "       -3.31882276e-02, -3.15022282e-02, -1.45274764e-02,  8.99626594e-03,\n",
       "       -6.49947580e-03,  9.82843041e-02,  2.28631198e-02,  5.08535933e-03,\n",
       "       -2.16242601e-03,  4.89163212e-03, -4.67054173e-02,  1.57190174e-01,\n",
       "        3.16710807e-02, -1.26277395e-02,  7.07610324e-03, -2.63007707e-03,\n",
       "        2.23008674e-02, -9.60244387e-02, -1.47624705e-02,  2.65120566e-02,\n",
       "        1.32904397e-02, -1.60293505e-02, -1.32097974e-02,  3.95276584e-02,\n",
       "       -3.52856144e-02,  3.37307863e-02,  9.93523933e-03,  2.27543954e-02,\n",
       "        1.05282441e-02, -5.10139726e-02, -4.19330969e-02,  2.03522015e-03,\n",
       "       -4.30715419e-02,  1.05101187e-02,  7.79297128e-02,  2.96756998e-02,\n",
       "        1.23835076e-02,  3.20845172e-02,  2.04693601e-02, -3.18663605e-02,\n",
       "        1.71557814e-02,  5.61856711e-03,  1.22987134e-02, -3.75122353e-02,\n",
       "        1.58035439e-02,  4.20127511e-02, -1.24607319e-02,  3.42583656e-02,\n",
       "        1.47952363e-02,  1.96049288e-02,  3.51178497e-02, -4.43438776e-02,\n",
       "       -3.55698615e-02, -1.18084647e-01, -1.00666157e-03, -2.83452566e-03,\n",
       "       -9.33585763e-02,  1.95232071e-02, -5.28742969e-02, -2.80063599e-02,\n",
       "       -1.26961973e-02,  1.29922489e-02,  2.80095115e-02, -4.81321849e-02,\n",
       "        2.62818332e-07,  1.80893131e-02, -1.51711497e-02, -2.03897133e-02,\n",
       "       -2.84580346e-02, -1.53550673e-02, -1.79113969e-02, -3.40540670e-02,\n",
       "       -3.72613519e-02,  2.08138917e-02,  1.77255669e-03,  5.16345873e-02,\n",
       "       -3.60502452e-02,  3.99684235e-02, -3.66573757e-03,  4.61929739e-02,\n",
       "       -4.03546132e-02, -8.97465050e-02, -4.64367978e-02, -6.66128844e-03,\n",
       "       -5.00133447e-03,  4.43970561e-02, -2.43180282e-02, -5.87579003e-03,\n",
       "       -1.45340487e-02, -5.29031903e-02,  1.80853792e-02, -1.99119858e-02,\n",
       "       -2.57812478e-02, -2.94113848e-02, -5.17488867e-02,  1.84630491e-02,\n",
       "       -3.06568351e-02, -3.43142264e-02,  2.95383297e-02, -2.71990225e-02,\n",
       "       -1.30143100e-02, -1.98793001e-02,  7.36410636e-03, -3.33207659e-02,\n",
       "       -2.57985126e-02, -2.69795656e-02,  1.96268540e-02, -2.51147393e-02,\n",
       "        7.82277137e-02,  1.40251229e-02, -7.41123632e-02, -2.08707098e-02,\n",
       "       -5.03862649e-03,  1.86584089e-02,  5.48126809e-02, -1.39224331e-03,\n",
       "        4.44329670e-03, -5.92845157e-02,  4.64762673e-02, -4.53331799e-04,\n",
       "        3.33899930e-02, -1.86654553e-02, -7.08509143e-03,  4.14489629e-03,\n",
       "       -5.31608388e-02, -6.44606799e-02,  3.18996795e-02, -1.43256355e-02,\n",
       "        3.03740185e-02, -1.61513798e-02,  4.19654287e-02, -1.29915448e-02,\n",
       "        1.95603316e-34, -7.69059826e-03,  1.68949906e-02, -2.42807027e-02,\n",
       "        4.99699526e-02,  2.19596103e-02, -2.34986632e-03,  3.29641104e-02,\n",
       "        8.29521660e-03, -3.21464650e-02, -7.35378964e-03, -5.13967164e-02],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings[122]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee4cadb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#import nltk\n",
    "#from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f224b6fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load SpaCy English Model\n",
    "#nlp = en_core_web_md.load()\n",
    "\n",
    "#Tags to remove\n",
    "#extags = ['PRON','CCONJ','PUNCT','PART','DET','ADP','NUM','SYM','SPACE']\n",
    "\n",
    "#tokens=[]\n",
    "#SpaCy tokenization + lemmatization + lowercase\n",
    "#for speech in nlp.pipe(speeches):\n",
    "#    scr_tok = [token.lemma_.lower() for token in speech if token.pos_ not in extags and not token.is_stop and token.is_alpha]\n",
    "#    tokens.append(scr_tok)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dcb2667",
   "metadata": {},
   "source": [
    "<A HREF=\"https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html#removing-stop-words\">Removing stop words</A> a little different than sometimes\n",
    "<P>\n",
    "At times, stop words might end up in our topic representations. This is something we typically want to avoid as they contribute little to the interpretation of the topics. However, removing stop words as a preprocessing step is not advised as the transformer-based embedding models that we use need the full context in order to create accurate embeddings.\n",
    "</P>\n",
    "<P>\n",
    "Instead, we can use the CountVectorizer to preprocess our documents after having generated embeddings and clustered our documents. Personally, I have found almost no disadvantages to using the CountVectorizer to remove stopwords and it is something I would strongly advise to try out:\n",
    "</P>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d5bbd52",
   "metadata": {},
   "source": [
    "<A HREF=\"https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\">sklearn CountVectorizer docs</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "3097784a",
   "metadata": {},
   "outputs": [],
   "source": [
    "################# RUN THIS AFTER SENTENCE EMBEDDINGS ###################\n",
    "# This takes a minute or two, shorter than above\n",
    "stop_words = text.ENGLISH_STOP_WORDS.union(['thats', 'just', 'im', 'did', 'thing', 'mr', 'al', 'thank', 'okay', 'thanks', 'question'])\n",
    "#vectorizer_model = CountVectorizer(stop_words='english', ngram_range=(1,3))\n",
    "vectorizer_model = CountVectorizer(stop_words=stop_words, ngram_range=(1,3))\n",
    "model = BERTopic(vectorizer_model=vectorizer_model, nr_topics=15)\n",
    "topics, probs = model.fit_transform(sentences, embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "140feb6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Count</th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>73349</td>\n",
       "      <td>-1_people_going_make_know</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2214</td>\n",
       "      <td>0_health_care_insurance_health care</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1580</td>\n",
       "      <td>1_nuclear_iran_weapons_sanctions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1085</td>\n",
       "      <td>2_congress_senate_republicans_bipartisan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>852</td>\n",
       "      <td>3_president_president president_obama_presiden...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>743</td>\n",
       "      <td>4_right_true_right right_young lady</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5</td>\n",
       "      <td>628</td>\n",
       "      <td>5_israel_palestinians_israels_israelis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>6</td>\n",
       "      <td>578</td>\n",
       "      <td>6_governor_romney_governor romney_ms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>7</td>\n",
       "      <td>572</td>\n",
       "      <td>7_afghanistan_afghan_troops_iraq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8</td>\n",
       "      <td>548</td>\n",
       "      <td>8_cuba_cuban_cuban people_cubans</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>9</td>\n",
       "      <td>536</td>\n",
       "      <td>9_black_rights_african_civil rights</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>10</td>\n",
       "      <td>520</td>\n",
       "      <td>10_climate_change_climate change_carbon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11</td>\n",
       "      <td>489</td>\n",
       "      <td>11_seat_everybody_everybody seat_hello</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12</td>\n",
       "      <td>483</td>\n",
       "      <td>12_snob___</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>13</td>\n",
       "      <td>466</td>\n",
       "      <td>13_everybody_work_everybody everybody_responsi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>14</td>\n",
       "      <td>465</td>\n",
       "      <td>14_tax_taxes_percent_unemployment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Topic  Count                                               Name\n",
       "0      -1  73349                          -1_people_going_make_know\n",
       "1       0   2214                0_health_care_insurance_health care\n",
       "2       1   1580                   1_nuclear_iran_weapons_sanctions\n",
       "3       2   1085           2_congress_senate_republicans_bipartisan\n",
       "4       3    852  3_president_president president_obama_presiden...\n",
       "5       4    743                4_right_true_right right_young lady\n",
       "6       5    628             5_israel_palestinians_israels_israelis\n",
       "7       6    578               6_governor_romney_governor romney_ms\n",
       "8       7    572                   7_afghanistan_afghan_troops_iraq\n",
       "9       8    548                   8_cuba_cuban_cuban people_cubans\n",
       "10      9    536                9_black_rights_african_civil rights\n",
       "11     10    520            10_climate_change_climate change_carbon\n",
       "12     11    489             11_seat_everybody_everybody seat_hello\n",
       "13     12    483                                         12_snob___\n",
       "14     13    466  13_everybody_work_everybody everybody_responsi...\n",
       "15     14    465                  14_tax_taxes_percent_unemployment"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_topic_info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86a387ab",
   "metadata": {},
   "source": [
    "The -1 topic is an outlier topic and contains documents that are generally unrelated to one another. The documents in that topic are essentially random in nature and therefore do not represent an actual topic. Thus, the -1 topic cannot be interpreted as all other topics. <A HREF=\"https://github.com/MaartenGr/BERTopic/issues/677\">reference</A>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "641a924f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('health', 0.05743176192986177),\n",
       " ('care', 0.04968332592662061),\n",
       " ('insurance', 0.04799788859926906),\n",
       " ('health care', 0.039829151255246255),\n",
       " ('health insurance', 0.023812967195562845),\n",
       " ('medicare', 0.01807584033017865),\n",
       " ('costs', 0.015565726202935826),\n",
       " ('people', 0.015555627819126698),\n",
       " ('affordable', 0.015325133189273666),\n",
       " ('coverage', 0.014518833270966312)]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_topic(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "0878852d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{-1: [('people', 0.010377612906681204),\n",
       "  ('going', 0.007641722375852682),\n",
       "  ('make', 0.006784588271381897),\n",
       "  ('know', 0.006144650102223625),\n",
       "  ('think', 0.006119250472113379),\n",
       "  ('want', 0.006080408040338213),\n",
       "  ('president', 0.006072376561537916),\n",
       "  ('world', 0.006019679118082326),\n",
       "  ('like', 0.005879196322329415),\n",
       "  ('weve', 0.005649466781900936)],\n",
       " 0: [('health', 0.05803863125973147),\n",
       "  ('care', 0.05051548908254219),\n",
       "  ('insurance', 0.047716179074347706),\n",
       "  ('health care', 0.040178550113369065),\n",
       "  ('health insurance', 0.023945236359090027),\n",
       "  ('medicare', 0.018183646898372544),\n",
       "  ('affordable', 0.015372234813416168),\n",
       "  ('people', 0.015291624693165876),\n",
       "  ('costs', 0.015228378211031887),\n",
       "  ('coverage', 0.014164132671861066)],\n",
       " 1: [('nuclear', 0.07177077407580301),\n",
       "  ('iran', 0.061960487986230904),\n",
       "  ('weapons', 0.02437270120221643),\n",
       "  ('sanctions', 0.023082420766392067),\n",
       "  ('iranian', 0.02186307434905729),\n",
       "  ('nuclear weapons', 0.021213822794592548),\n",
       "  ('deal', 0.0188400329616135),\n",
       "  ('weapon', 0.018281018324361572),\n",
       "  ('irans', 0.017990511839660415),\n",
       "  ('nuclear weapon', 0.017830876256464866)],\n",
       " 2: [('congress', 0.05007916048939116),\n",
       "  ('senate', 0.020728843470459182),\n",
       "  ('republicans', 0.02050234386489259),\n",
       "  ('bipartisan', 0.020238240897682388),\n",
       "  ('immigration', 0.0169620391926045),\n",
       "  ('members', 0.015439417048896375),\n",
       "  ('house', 0.015311047756006353),\n",
       "  ('republican', 0.014058010379213443),\n",
       "  ('democrats', 0.013439746243230632),\n",
       "  ('pass', 0.01238827734226621)],\n",
       " 3: [('president', 0.31196775687428263),\n",
       "  ('president president', 0.21044327641052069),\n",
       "  ('obama', 0.20188759506314424),\n",
       "  ('president obama', 0.19423709115996138),\n",
       "  ('president president president', 0.11735735320911407),\n",
       "  ('president president obama', 0.09395599944544106),\n",
       "  ('obama president', 0.05948044734403981),\n",
       "  ('president obama president', 0.053058602695845114),\n",
       "  ('obama president obama', 0.03056176626632291),\n",
       "  ('audience', 0.029531145721016228)],\n",
       " 4: [('right', 0.22607689875654327),\n",
       "  ('true', 0.10236687447830195),\n",
       "  ('right right', 0.09430468193666904),\n",
       "  ('young lady', 0.055729703090291155),\n",
       "  ('turn', 0.05222706666156383),\n",
       "  ('lady', 0.051141685374872986),\n",
       "  ('young lady right', 0.04690373647956366),\n",
       "  ('lady right', 0.04690373647956366),\n",
       "  ('young', 0.043634612880803815),\n",
       "  ('wrong', 0.038751740671883035)],\n",
       " 5: [('israel', 0.08371930244415854),\n",
       "  ('palestinians', 0.03758233692390678),\n",
       "  ('israels', 0.03574152357796213),\n",
       "  ('israelis', 0.031515893561092204),\n",
       "  ('peace', 0.02995779148028254),\n",
       "  ('palestinian', 0.02911334891797962),\n",
       "  ('security', 0.02395963475563671),\n",
       "  ('state', 0.02198200664449818),\n",
       "  ('israeli', 0.02088017706881238),\n",
       "  ('jewish', 0.017516435042783062)],\n",
       " 6: [('governor', 0.17795187384681024),\n",
       "  ('romney', 0.1538056879842703),\n",
       "  ('governor romney', 0.14979464655502597),\n",
       "  ('ms', 0.05827493541505777),\n",
       "  ('lehrer', 0.04151360117834953),\n",
       "  ('mayor', 0.03382235153310404),\n",
       "  ('schieffer', 0.027599433974825293),\n",
       "  ('ms ms', 0.02361958443405401),\n",
       "  ('president', 0.018320873381148347),\n",
       "  ('governor governor', 0.016800921528557194)],\n",
       " 7: [('afghanistan', 0.07569749033205647),\n",
       "  ('afghan', 0.06332466491580527),\n",
       "  ('troops', 0.04681029200691047),\n",
       "  ('iraq', 0.0322621326159354),\n",
       "  ('forces', 0.028279170928184837),\n",
       "  ('afghans', 0.02524331849258325),\n",
       "  ('security', 0.021911466376539318),\n",
       "  ('end', 0.02072002589261751),\n",
       "  ('war', 0.019837614098512313),\n",
       "  ('taliban', 0.019368989811953563)],\n",
       " 8: [('cuba', 0.08906414007831183),\n",
       "  ('cuban', 0.0763904395620698),\n",
       "  ('cuban people', 0.037379665957593816),\n",
       "  ('cubans', 0.030888202723538087),\n",
       "  ('castro', 0.030512785618683307),\n",
       "  ('people', 0.024957628676182602),\n",
       "  ('president castro', 0.020934669830979177),\n",
       "  ('united states', 0.0198443613493232),\n",
       "  ('united', 0.01964294268726774),\n",
       "  ('states', 0.019238440632415345)],\n",
       " 9: [('black', 0.032169309748384264),\n",
       "  ('rights', 0.025511640334097297),\n",
       "  ('african', 0.023541084219104058),\n",
       "  ('civil rights', 0.022078063447042592),\n",
       "  ('civil', 0.020185228121865444),\n",
       "  ('slavery', 0.01611304407769833),\n",
       "  ('white', 0.01569980585196749),\n",
       "  ('african american', 0.015188088550556091),\n",
       "  ('selma', 0.01421437884828421),\n",
       "  ('racial', 0.013180757433647999)],\n",
       " 10: [('climate', 0.08444073384412236),\n",
       "  ('change', 0.08146769165837499),\n",
       "  ('climate change', 0.0675577246301035),\n",
       "  ('carbon', 0.03125314422670217),\n",
       "  ('emissions', 0.023770830228501825),\n",
       "  ('planet', 0.01812658301463325),\n",
       "  ('agreement', 0.017208781256723715),\n",
       "  ('pollution', 0.015945165532918863),\n",
       "  ('countries', 0.015796406190591833),\n",
       "  ('paris', 0.015086670196264742)],\n",
       " 11: [('seat', 0.17303595821281403),\n",
       "  ('everybody', 0.11486953397922894),\n",
       "  ('everybody seat', 0.10261179041774686),\n",
       "  ('hello', 0.09163309811146234),\n",
       "  ('wheres', 0.08438915562135614),\n",
       "  ('right everybody', 0.06195776241230927),\n",
       "  ('right', 0.060693165956461424),\n",
       "  ('sir', 0.0536589313348339),\n",
       "  ('hello everybody', 0.04909360186916064),\n",
       "  ('seat seat', 0.04822693112455122)],\n",
       " 12: [('snob', 11.769256179512112),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05),\n",
       "  ('', 1e-05)],\n",
       " 13: [('everybody', 0.20885369814405264),\n",
       "  ('work', 0.10816418040998534),\n",
       "  ('everybody everybody', 0.10157083071862132),\n",
       "  ('responsibility', 0.06673583707638442),\n",
       "  ('got', 0.06345553356342755),\n",
       "  ('work everybody', 0.05739625180135082),\n",
       "  ('weve got', 0.05375973733426491),\n",
       "  ('weve', 0.04360333841920299),\n",
       "  ('weve got work', 0.03928771455214753),\n",
       "  ('got work', 0.03817819866955677)],\n",
       " 14: [('tax', 0.08637841870683889),\n",
       "  ('taxes', 0.0484273341322739),\n",
       "  ('percent', 0.04282355130906846),\n",
       "  ('unemployment', 0.02798362603874847),\n",
       "  ('rate', 0.02450995930160076),\n",
       "  ('cut', 0.021075625111982785),\n",
       "  ('loopholes', 0.0205789435319241),\n",
       "  ('tax code', 0.02041348485812544),\n",
       "  ('pay', 0.019674257255058655),\n",
       "  ('code', 0.019672203446365687)]}"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "fd0e753c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('health', 0.05743176192986177),\n",
       " ('care', 0.04968332592662061),\n",
       " ('insurance', 0.04799788859926906),\n",
       " ('health care', 0.039829151255246255),\n",
       " ('health insurance', 0.023812967195562845),\n",
       " ('medicare', 0.01807584033017865),\n",
       " ('costs', 0.015565726202935826),\n",
       " ('people', 0.015555627819126698),\n",
       " ('affordable', 0.015325133189273666),\n",
       " ('coverage', 0.014518833270966312)]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_topic(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "e4dd3190",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\io\\_renderers.py:395: DeprecationWarning:\n",
      "\n",
      "distutils Version classes are deprecated. Use packaging.version instead.\n",
      "\n",
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\plotly\\io\\_renderers.py:395: DeprecationWarning:\n",
      "\n",
      "distutils Version classes are deprecated. Use packaging.version instead.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "customdata": [
          [
           0,
           "health | care | insurance | health care | health insurance",
           2166
          ],
          [
           1,
           "nuclear | iran | weapons | sanctions | iranian",
           1594
          ],
          [
           2,
           "education | college | students | schools | teachers",
           1332
          ],
          [
           3,
           "president | obama | president obama | president president | president president president",
           1307
          ],
          [
           4,
           "congress | republicans | senate | bipartisan | house",
           1021
          ],
          [
           5,
           "shes | woman | said | parks | years",
           996
          ],
          [
           6,
           "governor | romney | governor romney | ms | lehrer",
           707
          ],
          [
           7,
           "israel | palestinians | israels | israelis | palestinian",
           695
          ],
          [
           8,
           "qualified | hes | highly qualified | qualifications | highly",
           644
          ],
          [
           9,
           "energy | clean | clean energy | oil | solar",
           632
          ],
          [
           10,
           "afghanistan | afghan | troops | iraq | forces",
           586
          ],
          [
           11,
           "change | climate | climate change | carbon | emissions",
           578
          ],
          [
           12,
           "deficit | spending | budget | cuts | trillion",
           578
          ],
          [
           13,
           "cuba | cuban | cuban people | cubans | castro",
           544
          ],
          [
           14,
           "right | right right | true | right right right | young lady",
           524
          ]
         ],
         "hovertemplate": "<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}",
         "legendgroup": "",
         "marker": {
          "color": "#B0BEC5",
          "line": {
           "color": "DarkSlateGrey",
           "width": 2
          },
          "size": [
           2166,
           1594,
           1332,
           1307,
           1021,
           996,
           707,
           695,
           644,
           632,
           586,
           578,
           578,
           544,
           524
          ],
          "sizemode": "area",
          "sizeref": 1.35375,
          "symbol": "circle"
         },
         "mode": "markers",
         "name": "",
         "orientation": "v",
         "showlegend": false,
         "type": "scatter",
         "x": [
          15.897143363952637,
          2.302208423614502,
          16.210586547851562,
          -11.26292896270752,
          17.10965919494629,
          16.534086227416992,
          -11.09358024597168,
          2.2713661193847656,
          -10.62195110321045,
          19.41680908203125,
          2.9065639972686768,
          19.694950103759766,
          17.50186538696289,
          2.5847415924072266,
          -10.99608039855957
         ],
         "xaxis": "x",
         "y": [
          9.4109525680542,
          2.5912985801696777,
          9.1500883102417,
          -0.5408980250358582,
          9.558489799499512,
          9.437396049499512,
          -0.12329202145338058,
          3.0319578647613525,
          -0.910537600517273,
          -13.081646919250488,
          2.796297311782837,
          -13.359782218933105,
          9.41523265838623,
          3.45654034614563,
          -0.9732624292373657
         ],
         "yaxis": "y"
        }
       ],
       "layout": {
        "annotations": [
         {
          "showarrow": false,
          "text": "D1",
          "x": -12.952368307113648,
          "y": -2.185743141174317,
          "yshift": 10
         },
         {
          "showarrow": false,
          "text": "D2",
          "x": 4.8484121561050415,
          "xshift": 10,
          "y": 10.992263269424438
         }
        ],
        "height": 650,
        "hoverlabel": {
         "bgcolor": "white",
         "font": {
          "family": "Rockwell",
          "size": 16
         }
        },
        "legend": {
         "itemsizing": "constant",
         "tracegroupgap": 0
        },
        "margin": {
         "t": 60
        },
        "shapes": [
         {
          "line": {
           "color": "#CFD8DC",
           "width": 2
          },
          "type": "line",
          "x0": 4.8484121561050415,
          "x1": 4.8484121561050415,
          "y0": -15.363749551773072,
          "y1": 10.992263269424438
         },
         {
          "line": {
           "color": "#9E9E9E",
           "width": 2
          },
          "type": "line",
          "x0": -12.952368307113648,
          "x1": 22.64919261932373,
          "y0": -2.185743141174317,
          "y1": -2.185743141174317
         }
        ],
        "sliders": [
         {
          "active": 0,
          "pad": {
           "t": 50
          },
          "steps": [
           {
            "args": [
             {
              "marker.color": [
               [
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 0",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 1",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 2",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 3",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 4",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 5",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 6",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 7",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 8",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 9",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 10",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 11",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 12",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red",
                "#B0BEC5"
               ]
              ]
             }
            ],
            "label": "Topic 13",
            "method": "update"
           },
           {
            "args": [
             {
              "marker.color": [
               [
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "#B0BEC5",
                "red"
               ]
              ]
             }
            ],
            "label": "Topic 14",
            "method": "update"
           }
          ]
         }
        ],
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "rgb(36,36,36)"
            },
            "error_y": {
             "color": "rgb(36,36,36)"
            },
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "baxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.6
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "rgb(237,237,237)"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "rgb(217,217,217)"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 1,
            "tickcolor": "rgb(36,36,36)",
            "ticks": "outside"
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "rgb(103,0,31)"
            ],
            [
             0.1,
             "rgb(178,24,43)"
            ],
            [
             0.2,
             "rgb(214,96,77)"
            ],
            [
             0.3,
             "rgb(244,165,130)"
            ],
            [
             0.4,
             "rgb(253,219,199)"
            ],
            [
             0.5,
             "rgb(247,247,247)"
            ],
            [
             0.6,
             "rgb(209,229,240)"
            ],
            [
             0.7,
             "rgb(146,197,222)"
            ],
            [
             0.8,
             "rgb(67,147,195)"
            ],
            [
             0.9,
             "rgb(33,102,172)"
            ],
            [
             1,
             "rgb(5,48,97)"
            ]
           ],
           "sequential": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ]
          },
          "colorway": [
           "#1F77B4",
           "#FF7F0E",
           "#2CA02C",
           "#D62728",
           "#9467BD",
           "#8C564B",
           "#E377C2",
           "#7F7F7F",
           "#BCBD22",
           "#17BECF"
          ],
          "font": {
           "color": "rgb(36,36,36)"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "white",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "white",
          "polar": {
           "angularaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "radialaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "yaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "zaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           }
          },
          "shapedefaults": {
           "fillcolor": "black",
           "line": {
            "width": 0
           },
           "opacity": 0.3
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "baxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "caxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          }
         }
        },
        "title": {
         "font": {
          "color": "Black",
          "size": 22
         },
         "text": "<b>Intertopic Distance Map",
         "x": 0.5,
         "xanchor": "center",
         "y": 0.95,
         "yanchor": "top"
        },
        "width": 650,
        "xaxis": {
         "anchor": "y",
         "domain": [
          0,
          1
         ],
         "range": [
          -12.952368307113648,
          22.64919261932373
         ],
         "title": {
          "text": ""
         },
         "visible": false
        },
        "yaxis": {
         "anchor": "x",
         "domain": [
          0,
          1
         ],
         "range": [
          -15.363749551773072,
          10.992263269424438
         ],
         "title": {
          "text": ""
         },
         "visible": false
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"7f5e8ef0-a723-4350-9223-3a3d596f7dfa\" class=\"plotly-graph-div\" style=\"height:650px; width:650px;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"7f5e8ef0-a723-4350-9223-3a3d596f7dfa\")) {                    Plotly.newPlot(                        \"7f5e8ef0-a723-4350-9223-3a3d596f7dfa\",                        [{\"customdata\":[[0,\"health | care | insurance | health care | health insurance\",2166],[1,\"nuclear | iran | weapons | sanctions | iranian\",1594],[2,\"education | college | students | schools | teachers\",1332],[3,\"president | obama | president obama | president president | president president president\",1307],[4,\"congress | republicans | senate | bipartisan | house\",1021],[5,\"shes | woman | said | parks | years\",996],[6,\"governor | romney | governor romney | ms | lehrer\",707],[7,\"israel | palestinians | israels | israelis | palestinian\",695],[8,\"qualified | hes | highly qualified | qualifications | highly\",644],[9,\"energy | clean | clean energy | oil | solar\",632],[10,\"afghanistan | afghan | troops | iraq | forces\",586],[11,\"change | climate | climate change | carbon | emissions\",578],[12,\"deficit | spending | budget | cuts | trillion\",578],[13,\"cuba | cuban | cuban people | cubans | castro\",544],[14,\"right | right right | true | right right right | young lady\",524]],\"hovertemplate\":\"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#B0BEC5\",\"size\":[2166,1594,1332,1307,1021,996,707,695,644,632,586,578,578,544,524],\"sizemode\":\"area\",\"sizeref\":1.35375,\"symbol\":\"circle\",\"line\":{\"color\":\"DarkSlateGrey\",\"width\":2}},\"mode\":\"markers\",\"name\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"x\":[15.897143363952637,2.302208423614502,16.210586547851562,-11.26292896270752,17.10965919494629,16.534086227416992,-11.09358024597168,2.2713661193847656,-10.62195110321045,19.41680908203125,2.9065639972686768,19.694950103759766,17.50186538696289,2.5847415924072266,-10.99608039855957],\"xaxis\":\"x\",\"y\":[9.4109525680542,2.5912985801696777,9.1500883102417,-0.5408980250358582,9.558489799499512,9.437396049499512,-0.12329202145338058,3.0319578647613525,-0.910537600517273,-13.081646919250488,2.796297311782837,-13.359782218933105,9.41523265838623,3.45654034614563,-0.9732624292373657],\"yaxis\":\"y\",\"type\":\"scatter\"}],                        {\"template\":{\"data\":{\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"bar\":[{\"error_x\":{\"color\":\"rgb(36,36,36)\"},\"error_y\":{\"color\":\"rgb(36,36,36)\"},\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"baxis\":{\"endlinecolor\":\"rgb(36,36,36)\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"rgb(36,36,36)\"},\"type\":\"carpet\"}],\"choropleth\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"choropleth\"}],\"contourcarpet\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"contourcarpet\"}],\"contour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"contour\"}],\"heatmapgl\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmapgl\"}],\"heatmap\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"heatmap\"}],\"histogram2dcontour\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2dcontour\"}],\"histogram2d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"histogram2d\"}],\"histogram\":[{\"marker\":{\"line\":{\"color\":\"white\",\"width\":0.6}},\"type\":\"histogram\"}],\"mesh3d\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"type\":\"mesh3d\"}],\"parcoords\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"parcoords\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}],\"scatter3d\":[{\"line\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatter3d\"}],\"scattercarpet\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattercarpet\"}],\"scattergeo\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergeo\"}],\"scattergl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattergl\"}],\"scattermapbox\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scattermapbox\"}],\"scatterpolargl\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolargl\"}],\"scatterpolar\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterpolar\"}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"scatterternary\":[{\"marker\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"type\":\"scatterternary\"}],\"surface\":[{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"},\"colorscale\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"type\":\"surface\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"rgb(237,237,237)\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"rgb(217,217,217)\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}]},\"layout\":{\"annotationdefaults\":{\"arrowhead\":0,\"arrowwidth\":1},\"autotypenumbers\":\"strict\",\"coloraxis\":{\"colorbar\":{\"outlinewidth\":1,\"tickcolor\":\"rgb(36,36,36)\",\"ticks\":\"outside\"}},\"colorscale\":{\"diverging\":[[0.0,\"rgb(103,0,31)\"],[0.1,\"rgb(178,24,43)\"],[0.2,\"rgb(214,96,77)\"],[0.3,\"rgb(244,165,130)\"],[0.4,\"rgb(253,219,199)\"],[0.5,\"rgb(247,247,247)\"],[0.6,\"rgb(209,229,240)\"],[0.7,\"rgb(146,197,222)\"],[0.8,\"rgb(67,147,195)\"],[0.9,\"rgb(33,102,172)\"],[1.0,\"rgb(5,48,97)\"]],\"sequential\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]],\"sequentialminus\":[[0.0,\"#440154\"],[0.1111111111111111,\"#482878\"],[0.2222222222222222,\"#3e4989\"],[0.3333333333333333,\"#31688e\"],[0.4444444444444444,\"#26828e\"],[0.5555555555555556,\"#1f9e89\"],[0.6666666666666666,\"#35b779\"],[0.7777777777777778,\"#6ece58\"],[0.8888888888888888,\"#b5de2b\"],[1.0,\"#fde725\"]]},\"colorway\":[\"#1F77B4\",\"#FF7F0E\",\"#2CA02C\",\"#D62728\",\"#9467BD\",\"#8C564B\",\"#E377C2\",\"#7F7F7F\",\"#BCBD22\",\"#17BECF\"],\"font\":{\"color\":\"rgb(36,36,36)\"},\"geo\":{\"bgcolor\":\"white\",\"lakecolor\":\"white\",\"landcolor\":\"white\",\"showlakes\":true,\"showland\":true,\"subunitcolor\":\"white\"},\"hoverlabel\":{\"align\":\"left\"},\"hovermode\":\"closest\",\"mapbox\":{\"style\":\"light\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"white\",\"polar\":{\"angularaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"radialaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"zaxis\":{\"backgroundcolor\":\"white\",\"gridcolor\":\"rgb(232,232,232)\",\"gridwidth\":2,\"linecolor\":\"rgb(36,36,36)\",\"showbackground\":true,\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}},\"shapedefaults\":{\"fillcolor\":\"black\",\"line\":{\"width\":0},\"opacity\":0.3},\"ternary\":{\"aaxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"baxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"},\"bgcolor\":\"white\",\"caxis\":{\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\"}},\"title\":{\"x\":0.05},\"xaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"},\"yaxis\":{\"automargin\":true,\"gridcolor\":\"rgb(232,232,232)\",\"linecolor\":\"rgb(36,36,36)\",\"showgrid\":false,\"showline\":true,\"ticks\":\"outside\",\"title\":{\"standoff\":15},\"zeroline\":false,\"zerolinecolor\":\"rgb(36,36,36)\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[-12.952368307113648,22.64919261932373]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"\"},\"visible\":false,\"range\":[-15.363749551773072,10.992263269424438]},\"legend\":{\"tracegroupgap\":0,\"itemsizing\":\"constant\"},\"margin\":{\"t\":60},\"title\":{\"font\":{\"size\":22,\"color\":\"Black\"},\"text\":\"<b>Intertopic Distance Map\",\"y\":0.95,\"x\":0.5,\"xanchor\":\"center\",\"yanchor\":\"top\"},\"hoverlabel\":{\"font\":{\"size\":16,\"family\":\"Rockwell\"},\"bgcolor\":\"white\"},\"width\":650,\"height\":650,\"sliders\":[{\"active\":0,\"pad\":{\"t\":50},\"steps\":[{\"args\":[{\"marker.color\":[[\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 0\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 1\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 2\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 3\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 4\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 5\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 6\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 7\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 8\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 9\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 10\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 11\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\",\"#B0BEC5\"]]}],\"label\":\"Topic 12\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\",\"#B0BEC5\"]]}],\"label\":\"Topic 13\",\"method\":\"update\"},{\"args\":[{\"marker.color\":[[\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"#B0BEC5\",\"red\"]]}],\"label\":\"Topic 14\",\"method\":\"update\"}]}],\"shapes\":[{\"line\":{\"color\":\"#CFD8DC\",\"width\":2},\"type\":\"line\",\"x0\":4.8484121561050415,\"x1\":4.8484121561050415,\"y0\":-15.363749551773072,\"y1\":10.992263269424438},{\"line\":{\"color\":\"#9E9E9E\",\"width\":2},\"type\":\"line\",\"x0\":-12.952368307113648,\"x1\":22.64919261932373,\"y0\":-2.185743141174317,\"y1\":-2.185743141174317}],\"annotations\":[{\"showarrow\":false,\"text\":\"D1\",\"x\":-12.952368307113648,\"y\":-2.185743141174317,\"yshift\":10},{\"showarrow\":false,\"text\":\"D2\",\"x\":4.8484121561050415,\"xshift\":10,\"y\":10.992263269424438}]},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('7f5e8ef0-a723-4350-9223-3a3d596f7dfa');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.visualize_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "072b620f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\peter\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\scipy\\sparse\\_index.py:146: SparseEfficiencyWarning:\n",
      "\n",
      "Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Save Model\n",
    "model.save(\"obamaspeechmodel\")\n",
    "# Saved model is about 1GB... may not push to github, we'll see, maybe save it outside repository or add to .ignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a53d3c41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Model\n",
    "model = BERTopic.load(\"obamaspeechmodel\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "a919d99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "testfile = './Data/NYTimes/12-14-2012-sandyhook.txt'\n",
    "with open(testfile, encoding='utf-8') as f:\n",
    "        test = f.read()\n",
    "f.close()\n",
    "test = test.replace(\"\\n\\n\", \"\")\n",
    "test = test.split('.')\n",
    "testembed = sentence_model.encode(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "ea1d5fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "testtopics, testprobs = model.transform(test, testembed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "52e80b61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('health', 0.05803863125973147),\n",
       "  ('care', 0.05051548908254219),\n",
       "  ('insurance', 0.047716179074347706),\n",
       "  ('health care', 0.040178550113369065),\n",
       "  ('health insurance', 0.023945236359090027),\n",
       "  ('medicare', 0.018183646898372544),\n",
       "  ('affordable', 0.015372234813416168),\n",
       "  ('people', 0.015291624693165876),\n",
       "  ('costs', 0.015228378211031887),\n",
       "  ('coverage', 0.014164132671861066)],\n",
       " [('everybody', 0.20885369814405264),\n",
       "  ('work', 0.10816418040998534),\n",
       "  ('everybody everybody', 0.10157083071862132),\n",
       "  ('responsibility', 0.06673583707638442),\n",
       "  ('got', 0.06345553356342755),\n",
       "  ('work everybody', 0.05739625180135082),\n",
       "  ('weve got', 0.05375973733426491),\n",
       "  ('weve', 0.04360333841920299),\n",
       "  ('weve got work', 0.03928771455214753),\n",
       "  ('got work', 0.03817819866955677)],\n",
       " [('everybody', 0.20885369814405264),\n",
       "  ('work', 0.10816418040998534),\n",
       "  ('everybody everybody', 0.10157083071862132),\n",
       "  ('responsibility', 0.06673583707638442),\n",
       "  ('got', 0.06345553356342755),\n",
       "  ('work everybody', 0.05739625180135082),\n",
       "  ('weve got', 0.05375973733426491),\n",
       "  ('weve', 0.04360333841920299),\n",
       "  ('weve got work', 0.03928771455214753),\n",
       "  ('got work', 0.03817819866955677)],\n",
       " [('health', 0.05803863125973147),\n",
       "  ('care', 0.05051548908254219),\n",
       "  ('insurance', 0.047716179074347706),\n",
       "  ('health care', 0.040178550113369065),\n",
       "  ('health insurance', 0.023945236359090027),\n",
       "  ('medicare', 0.018183646898372544),\n",
       "  ('affordable', 0.015372234813416168),\n",
       "  ('people', 0.015291624693165876),\n",
       "  ('costs', 0.015228378211031887),\n",
       "  ('coverage', 0.014164132671861066)],\n",
       " [('health', 0.05803863125973147),\n",
       "  ('care', 0.05051548908254219),\n",
       "  ('insurance', 0.047716179074347706),\n",
       "  ('health care', 0.040178550113369065),\n",
       "  ('health insurance', 0.023945236359090027),\n",
       "  ('medicare', 0.018183646898372544),\n",
       "  ('affordable', 0.015372234813416168),\n",
       "  ('people', 0.015291624693165876),\n",
       "  ('costs', 0.015228378211031887),\n",
       "  ('coverage', 0.014164132671861066)]]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[model.get_topic(topic) for topic in testtopics if topic>=0]\n",
    "# There isn't a gun control topic so this is what it came up with when it wasn't spitting out -1 (outlier)\n",
    "# work/responsibility and health care"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c64dcab6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}