topic_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipediaapi\n",
    "import random\n",
    "import spacy\n",
    "import re\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import PCA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install dependencies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of keywords per topic\n",
    "history_keywords = ['Allied Powers', 'Axis Powers', 'Appeasement', 'Central Powers', 'Duma', 'Fascism', 'Great Depression', 'Holocaust', 'League of Nations', 'Mandate', 'Mexican Revolution', 'Pan-Slavic movement', 'Reparations', 'Revolution of  (Russia)', 'Russo-Japanese War', 'Spanish Civil War', 'Tahran Conference', 'Treaty of Versailles', 'United Nations', 'Yalta Conference ', 'Afrikaners', 'Apartheid', 'AyatollahKhomeini', 'Berlin Wall', 'Brinkmanship', 'Coalition', 'Cold War', 'Collectivization', 'Containment', 'Cuban Missile Crisis', 'CulturalRevolution', 'Five Year Plans', 'Geneva Conference', 'Genocide', 'Glasnost', 'Great Leap Forward', 'Guomindang', 'Iron Curtain', 'Marshall Plan ', 'May Fourth Movement', 'New Economic Policy (NEP)', 'Nonalignment', 'North Atlantic treaty Organization (NATO)', 'Prague Spring', 'Great Purges', 'Red Guard', 'Tiananmen Square', 'Truman Doctrine', 'Warsaw Pact ', 'Al-Qaeda', 'Cartels', 'InternationalMonetary Fund (IMF)', 'Persian Gulf War ', 'World Bank ', 'European Economic Community (EEC)', 'European Union/Euro', 'North American Free Trade Agreement (NAFTA)', 'Organization of Petroleum Exporting Countries (OPEC)', 'World Trade Organization (WTO)', 'Helsinki Accords', 'Evangelical ', 'Fundamentalism', 'Mass consumerism', 'New Deal', 'Welfare state', 'Green Revolution', 'Guest workers', 'Ozone depletion', 'Totalitarian', 'Tariff', 'Woodrow Wilson', 'Fourteen Points', 'Bolsheviks', 'Fascism', 'Adolf Hitler', 'Winston Churchill', 'Total War', 'Joseph Stalin', 'Lenin', 'Mao Zedong', 'Nikita Khrushchev', 'Mikhail Gorbachev', 'Deng Xiaoping', 'Mahatma Gandhi', 'Muslim League', 'Muhammad Ali Jinnah', 'African national Congress', 'Nelson Mandela ', 'Black Shirts', 'Berlin Wall']\n",
    "math_keywords = ['Absolute differential calculus', 'Absolute geometry', 'Abstract algebra', 'Abstract analytic number theory', 'Abstract differential geometry', 'Abstract harmonic analysis', 'Abstract homotopy theory', 'Actuarial science', 'Additive combinatorics', 'Additive number theory', 'Affine geometry', 'Affine geometry of curves', 'Affine differential geometry', 'Ahlfors theory', 'Algebra', 'Algebraic analysis', 'Algebraic combinatorics', 'Algebraic computation', 'Algebraic geometry', 'Algebraic graph theory', 'Algebraic K-theory', 'Algebraic number theory', 'Algebraic statistics', 'Algebraic topology', 'Algorithmic number theory', 'Anabelian geometry', 'Analysis', 'Analytic combinatorics', 'Analytic geometry', 'Analytic number theory', 'Applied mathematics', 'Approximation theory', 'Arakelov geometry', 'Arakelov theory', 'Arithmetic', 'Arithmetic algebraic geometry', 'Arithmetic combinatorics', 'Arithmetic dynamics', 'Arithmetic geometry', 'Arithmetic topology', 'Arithmetical algebraic geometry', 'Asymptotic combinatorics', 'Asymptotic theory', 'Auslander–Reiten theory', 'Axiomatic geometry', 'Axiomatic set theory', 'Bifurcation theory', 'Biostatistics', 'Birational geometry', 'Bolyai–Lobachevskian geometry', 'C*-algebra theory', 'Cartesian geometry', 'Calculus', 'Calculus of infinitesimals', 'Calculus of moving surfaces', 'Calculus of variations', 'Catastrophe theory', 'Categorical logic', 'Category theory', 'Chaos theory', 'Character theory', 'Class field theory', 'Classical differential geometry', 'Classical analysis', 'Classical Euclidean geometry', 'Classical geometry', 'Classical invariant theory', 'Classical mathematics', 'Clifford analysis', 'Coding theory', 'Combinatorial commutative algebra', 'Combinatorial design theory', 'Combinatorial geometry', 'Combinatorial group theory', 'Combinatorial mathematics', 'Combinatorial set theory', 'Combinatorial topology', 'Combinatorics', 'Commutative algebra', 'Complex algebraic geometry', 'Complex analysis', 'Complex analytic dynamics', 'Complex analytic geometry', 'Complex differential geometry', 'Complex dynamics', 'Complex geometry', 'Complexity theory', 'Computable analysis', 'Computable model theory', 'Computability theory', 'Computational complexity theory', 'Computational geometry', 'Computational group theory', 'Computational mathematics', 'Computational number theory', 'Computer algebra', 'Conformal geometry', 'Constructive analysis', 'Constructive function theory', 'Constructive mathematics', 'Constructive quantum field theory', 'Constructive set theory', 'Contact geometry', 'Convex analysis', 'Convex geometry', 'Coordinate geometry', 'CR geometry', 'Descriptive set theory', 'Differential algebraic geometry', 'Differential calculus', 'Differential Galois theory', 'Differential geometry', 'Differential geometry of curves', 'Differential geometry of surfaces', 'Differential topology', 'Diophantine geometry', 'Discrete geometry', 'Discrete mathematics', 'Discrete Morse theory', 'Donaldson theory', 'Dynamical systems theory', 'Econometrics', 'Effective descriptive set theory', 'Elementary algebra', 'Elementary arithmetic', 'Elementary mathematics', 'Elementary group theory', 'Elimination theory', 'Elliptic geometry', 'Enumerative combinatorics', 'Enumerative geometry', 'Ergodic Ramsey theory', 'Ergodic theory', 'Euclidean differential geometry', 'Euler calculus', 'Experimental mathematics', 'Extremal combinatorics', 'Extremal graph theory', 'Field theory', 'Finite model theory', 'Finsler geometry', 'Fourier analysis', 'Fractal geometry', 'Fractional calculus', 'Fractional dynamics', 'Fredholm theory', 'Function theory', 'Functional analysis', 'Functional calculus', 'Fuzzy mathematics', 'Fuzzy set theory', 'Galois cohomology', 'Galois theory', 'Galois geometry', 'Game theory', 'General topology', 'Generalized trigonometry', 'Geometric algebra', 'Geometric analysis', 'Geometric calculus', 'Geometric combinatorics', 'Geometric function theory', 'Geometric invariant theory', 'Geometric graph theory', 'Geometric group theory', 'Geometric measure theory', 'Geometric topology', 'Geometry', 'Geometry of numbers', 'Global analysis', 'Graph theory', 'Group-character theory', 'Group theory', 'Gyrotrigonometry', 'Hard analysis', 'Harmonic analysis', 'Higher category theory', 'Higher-dimensional algebra', 'Hodge theory', 'Holomorphic functional calculus', 'Homological algebra', 'Hyperbolic geometry', 'hyperbolic trigonometry', 'Hypercomplex analysis', 'Ideal theory', 'Idempotent analysis', 'Incidence geometry', 'Inconsistent mathematics', 'Infinitary combinatorics', 'Infinitesimal analysis', 'Infinitesimal calculus', 'Information geometry', 'Integral geometry', 'Intersection theory', 'Intuitionistic type theory', 'Invariant theory', 'Inventory theory', 'Inversive geometry', 'Inversive plane geometry', 'Itô calculus', 'Iwasawa theory', 'K-theory', 'K-homology', 'Kähler geometry', 'KK-theory', 'Klein geometry', 'Knot theory', 'Kummer theory', 'L-theory', 'Large deviations theory', 'Large sample theory', 'Lattice theory', 'Lie sphere geometry', 'Linear programming', 'Local algebra', 'Local class field theory', 'Low-dimensional topology', 'Malliavin calculus', 'Mathematical biology', 'Mathematical chemistry', 'Mathematical economics', 'Mathematical finance', 'Mathematical logic', 'Mathematical physics', 'Mathematical psychology', 'Mathematical sciences', 'Mathematical sociology', 'Mathematical statistics', 'Model theory', 'Modern algebra', 'Modern algebraic geometry', 'Modern invariant theory', 'Modular representation theory', 'Morse theory', 'Multilinear algebra', 'Multiplicative number theory', 'Multivariable calculus', 'Neutral geometry', 'Nevanlinna theory', 'Nielsen theory', 'Nonarchimedean dynamics', 'Noncommutative algebraic geometry', 'Noncommutative harmonic analysis', 'Number theory', 'Operad theory', 'Operator theory', 'Optimal control theory', 'Order theory', 'Ordered geometry', 'p-adic analysis', 'p-adic dynamics', 'Paraconsistent mathematics', 'Point-set topology', 'Polyhedral combinatorics', 'Projective geometry', 'Pseudo-Riemannian geometry', 'Pure mathematics', 'Quantum calculus', 'Quantum geometry', 'Ramsey theory', 'Real algebra', 'Real algebraic geometry', 'Real analysis', 'Recreational mathematics', 'Recursion theory', 'Representation theory', 'Ribbon theory', 'Riemannian geometry', 'Rough set theory', 'Scheme theory', 'Semialgebraic geometry', 'Single operator theory', 'Singularity theory', 'Smooth infinitesimal analysis', 'Spectral geometry', 'Spectral graph theory', 'Spectral theory', 'Spectral theory of ordinary differential equations', 'Spectrum continuation analysis', 'Spherical geometry', 'Spherical trigonometry', 'Statistics', 'Stochastic geometry', 'Surgery theory', 'Symbolic computation', 'Symplectic geometry', 'Synthetic differential geometry', 'Synthetic geometry', 'Systolic geometry', 'Systolic hyperbolic geometry', 'Tensor analysis', 'Tensor calculus', 'Tensor theory', 'Tessellation', 'Theoretical physics', 'Topological combinatorics', 'Transcendental number theory', 'Trigonometry', 'Tropical analysis', 'Twisted K-theory', 'Umbral calculus', 'Uncertainty theory', 'Universal algebra', 'Universal hyperbolic trigonometry', 'Vector algebra', 'Vector analysis', 'Vector calculus']\n",
    "philosophy_keywords = ['Absurdism', 'Actual idealism', 'Actualism', 'Advaita Vedanta', 'Aesthetic Realism', 'Aesthetics', 'African philosophy', 'Afrocentrism', 'Agential realism', 'Agnosticism', 'Agnostic theism', 'American philosophy', 'Anarchy', 'Animism', 'Antinatalism', 'Antinomianism — Anti-realism', 'Antireductionism', 'Analytic philosophy', 'Anarchism', 'Ancient philosophy', 'Anthropocentrism', 'Anomalous monism', 'Applied ethics', 'Aristotelianism', 'Asceticism', 'Atheism', 'Authoritarianism', 'Averroism', 'Avicennism', 'Axiology   Baptism', 'Bayesianism', 'Bioconservatism', 'Biosophy', 'Buddhist philosophy   Cartesianism', 'Categorical imperative', 'Charvaka', 'Chinese naturalism', 'Capitalism', 'Chinese philosophy', 'Christian existentialism', 'Christian humanism', 'Christian philosophy', 'Cognitivism', 'Communitarianism', 'Communism', 'Compatibilism and incompatibilism', 'Confirmation holism', 'Confucianism', 'Consequentialism', 'Conceptualism', 'Conservatism', 'Constructivist epistemology', 'Continental philosophy', 'Cosmopolitanism', 'Critical rationalism', 'Critical realism', 'Cynicism', 'Czech philosophy   Danish philosophy', 'Daoism', 'Deconstruction', 'Deism', 'Denialism', 'Deontology', 'Depressionism', 'Determinism', 'Dialectic', 'Dialectical materialism', 'Didacticism', 'Digital physics', 'Discordianism', 'Dualistic cosmology', 'Dvaita   Ecocentrism', 'Ecumenism', 'Egalitarianism', 'Egocentrism', 'Egoism', 'Eliminative materialism', 'Emotivism', 'Empiricism', 'Ephesian school', 'Epiphenomenalism', 'Epicureanism', 'Epistemological nihilism', 'Epistemology', 'Esotericism', 'Essentialism', 'Ethics', 'Eternalism', 'Eudaimonism', 'Existentialism', 'Externalism   Fallibilism', 'Fascism', 'Fatalism', 'Feminist philosophy', 'Filial piety', 'Foundationalism', 'Free will', 'Fundamentalism', '  German idealism', 'German philosophy', 'Gnosticism', 'Greek philosophy', '  Hedonism', 'Hegelianism', 'Hermeticism', 'Henotheism', 'Heterophenomenology', 'Hindu philosophy', 'Historical materialism', 'Historicism', 'Holism', 'Hongaku', 'Humanism', 'Humanistic naturalism', 'Hylozoism', '  Idealism', 'Identityism', 'Ideological criticism', 'Ignosticism', 'Illegalism', 'Illuminationism', 'Individualism', 'Indian logic', 'Indian philosophy', 'Indonesian philosophy', 'Induction', 'Informal logic', 'Innatism', 'Instrumental rationality', 'Instrumentalism', 'Interactionism (philosophy of mind)', 'Internalism and externalism', 'Intuitionism', 'Iranian philosophy', 'Irrealism', 'Islamic ethics', 'Islamic philosophy   Japanese philosophy', 'Jainism', 'Jewish philosophy', 'Jingoism', 'Juche', 'Judeo-Islamic philosophies', 'Just war theory', 'Kantianism', 'Korean philosophy', 'Legalism', 'Leninism', 'Liberalism', 'Libertarianism', 'Libertarianism', 'Logic', 'Logical atomism', 'Logical positivism', 'Logicians', 'Logicism', 'Logic in China', 'Logic in Islamic philosophy', 'Manichaeism', 'Maoism', 'Marxism', 'Marxist philosophy of nature', 'Materialism', 'Mathematicism', 'Maxim (philosophy)', 'Medical ethics', 'Medieval philosophy', 'Medievalism', 'Mentalism', 'Mereological nihilism', 'Merism', 'Meta-philosophy', 'Metaphysics', 'Meta-ethics', 'Milesian school', 'Mimamsa', 'Mind-body dualism', 'Misology', 'Modernism', 'Modern Islamic philosophy', 'Mohism', 'Monism', 'Moral absolutism', 'Moral realism', 'Moral relativism', 'Moral skepticism', 'Mysticism', '  Naïve realism', 'Naturalism', 'Negative utilitarianism', 'Neo-Confucianism', 'Neo-Hegelianism', 'Neo-Kantianism', 'Neoplatonism', 'Neopythagoreanism', 'Neo-Scholasticism', 'Neotaoism', 'Neuroethics', 'Neurophilosophy', 'Neurotheology', 'Neutral monism', 'New realism', 'New Thought', 'Nihilism', 'Nominalism', 'Nondualism', 'Non-philosophy', 'Non-theism', 'Nyaya', '  Objective idealism', 'Objectivism', 'Occasionalism', 'Ontology', 'Ontotheology', 'Open individualism', 'Organicism   Paganism', 'Pakistani philosophy', 'Pancritical rationalism', 'Pandeism', 'Panpsychism', 'Pantheism', 'Pataphysics', 'Perennial philosophy', 'Perfectionism', 'Peripatetic school', 'Personalism', 'Perspectivism', 'Pessimism', 'Phenomenalism', 'Phenomenology', 'Philosophical anthropology', 'Philosophical Satanism', 'Philosophy of archaeology', 'Philosophy of art', 'Philosophy of Arithmetic', 'Philosophy of artificial intelligence', 'Philosophy of action', 'Philosophy of biology', 'Philosophy of business', 'Philosophy of Common Sense', 'Philosophy of culture', 'Philosophy of color', 'The Philosophy of Chance', 'Philosophy of design', 'Philosophy of dialogue', 'Philosophy of eating', 'Philosophy of economics', 'Philosophy of education', 'Philosophy of engineering', 'Philosophy of film', 'Philosophy of futility', 'Philosophy of geography', 'Philosophy of healthcare', 'Philosophy of history', 'Philosophy of information', 'Philosophy of language', 'Philosophy of Law', 'Philosophy of logic', 'Philosophy of love', 'Philosophy of mathematics', 'Philosophy of mathematics education', 'Philosophy of mind', 'Philosophy of motion', 'Philosophy of music', 'Philosophy of nature', 'Philosophy of Natural Science', 'Philosophy of neuroscience', 'Philosophy of perception', 'Philosophy of philosophy', 'Philosophy of physics', 'Philosophy of psychology', 'Philosophy of psychiatry', 'Philosophy of religion', 'Philosophy of religious language', 'Philosophy of science', 'Philosophy of sex', 'Philosophy of self', 'Philosophy of social science', 'Philosophy of space and time', 'Philosophy of sport', 'Philosophy of statistics', 'Philosophy of thermal and statistical physics', 'Philosophy of war', 'Physicalism', 'Physical ontology', 'Platonic realism', 'Platonism', 'Pluralism', 'Political philosophy', 'Populism', 'Positivism', 'Postanalytic philosophy', 'Post-structuralism', 'Posthumanism', 'Post-materialism', 'Post-modernism', 'Postpositivism- Practical reason', 'Pragmatism', 'Praxis School', 'Probabilism', 'Presentism', 'Process philosophy', 'Progressivism', 'Property dualism', 'Pseudophilosophy', 'Psychological egoism', 'Pure practical reason', 'Pure reason', 'Pyrrhonian skepticism', 'Pyrrhonism', 'Pythagoreanism', '  Quantum mysticism', 'Quietism   Raëlism', 'Rastafari', 'Rationalism', 'Realism', 'Reconstructivism', 'Reductionism', 'Reductive materialism', 'Reformational philosophy', 'Relationalism', 'Relativism', 'Relevance logic', 'Religious humanism', 'Religious philosophy', 'Reliabilism', 'Renaissance humanism', 'Romanian philosophy', 'Romanticism', 'Russian cosmism', 'Russian philosophy   Sabellianism', 'Sankhya', 'Scotism', 'Scholasticism', 'Scientism', 'Secularism', 'Secular humanism', 'Semantic holism', 'Sensualism', 'Sexualism', 'Sexism', 'Shamanism', 'Sikhism', 'Singularitarianism', 'Skepticism', 'Skeptical theism', 'Socialism', 'Social philosophy', 'Solipsism', 'Sophism', 'Spiritualism', 'Stoicism', 'Structuralism', 'Subjective idealism', 'Subjectivism', 'Sufi metaphysics', 'Śūnyatā', 'Supersessionism', 'Synoptic philosophy', 'Systems philosophy', '  Taoism', 'Teleology', 'Tetralemma', 'Theistic finitism', 'Theism', 'Thelema', 'Theology', 'Theosophy', 'Thomism', 'Traditionalist School', 'Transcendent theosophy (al-Hikmat al-Muta’liyah)', 'Transcendental idealism', 'Transcendentalism', 'Transcendental perspectivism', 'Transhumanism', 'Transmodernism', 'Type physicalism  Ubuntu', 'Universalism', 'Utilitarian bioethics', 'Utilitarianism', 'Value pluralism', 'Value theory', 'Vedanta', 'Verificationism', 'Verism', 'Vienna Circle', 'Virtue ethics', 'Vitalism', 'Voluntaryism- Western philosophy', 'Wu wei', 'Zen', 'Zoroastrianism', 'Zurvanism']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_docs(keyword_list):\n",
    "    \"\"\"\n",
    "    Takes a list of keywords and returns a list of articles from wikipedia\n",
    "    that match the keywords.\n",
    "    \"\"\"\n",
    "    wiki = wikipediaapi.Wikipedia('en')\n",
    "    num_found = 0\n",
    "    doc_list, title_list = [], []\n",
    "    for keyword in keyword_list:\n",
    "        page_py = wiki.page(keyword)\n",
    "        if page_py.exists():\n",
    "            doc_list.append(page_py.summary)\n",
    "            title_list.append(page_py.title)\n",
    "            num_found+=1\n",
    "    print(\"Found \"+str(num_found)+\" articles out of \"+str(len(keyword_list)))\n",
    "    return doc_list, title_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "History articles: \n",
      "\n",
      "Found 77 articles out of 90\n",
      "Math articles: \n",
      "\n",
      "Found 287 articles out of 307\n",
      "Philosophy articles: \n",
      "\n",
      "Found 376 articles out of 392\n"
     ]
    }
   ],
   "source": [
    "print(\"History articles: \\n\")\n",
    "history_docs, history_titles = get_docs(history_keywords)\n",
    "print(\"Math articles: \\n\")\n",
    "math_docs, math_titles = get_docs(math_keywords)\n",
    "print(\"Philosophy articles: \\n\")\n",
    "philosophy_docs, philosophy_titles = get_docs(philosophy_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge into one dataset doc_list\n",
    "doc_list = history_docs+math_docs+philosophy_docs\n",
    "title_list = history_titles+math_titles+philosophy_titles\n",
    "label_list = [0 for _ in range(len(history_docs))] + [1 for _ in range(len(math_docs))] + [2 for _ in range(len(math_docs))]\n",
    "c = list(zip(doc_list, title_list, label_list))\n",
    "random.shuffle(c)\n",
    "doc_list, title_list, label_list = zip(*c)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "77\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import re\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "\n",
    "from nltk.stem import PorterStemmer\n",
    "from multiprocessing import Pool\n",
    "\n",
    "# Tokenization and Cleaning\n",
    "        \n",
    "\n",
    "def split_string(lines):\n",
    "    sent_tokens = []\n",
    "    for line in lines:\n",
    "        line = re.sub(\"\\n\", \"\", line)\n",
    "        line = re.sub(\"\\\\\\\\.\", \"\", line)\n",
    "        line = re.sub(\"===\", \"\", line)\n",
    "        for sent in sent_tokenize(line):\n",
    "            sent_tokens.append(sent)\n",
    "    return sent_tokens\n",
    "\n",
    "def one_hot_enc(tup):\n",
    "    ar, dic_size = tup\n",
    "    #print(ar)\n",
    "    n = np.max(ar) +1\n",
    "    #count = np.sum(np.eye(dic_size)[ar], axis= 0)\n",
    "    count = np.sum(tf.one_hot(ar, n), axis= 0)/len(ar)\n",
    "    bl = np.array(count, dtype=bool)\n",
    "    ex = bl.astype(int)\n",
    "    \n",
    "    #padded = pad_sequences(oh, maxlen=x_voc_size, padding='post', truncating='post')\n",
    "    return count, ex\n",
    "\n",
    "def one_hot_all(input, dic_size):\n",
    "    #print(len(x_train))\n",
    "    #print(len(x_train[0]))\n",
    "    #print(x_train[0])\n",
    "    print(\"one hot all\")\n",
    "    existence = []\n",
    "    occurrence = []\n",
    "    p = Pool()\n",
    "    vector_input = [(each, dic_size) for each in input]\n",
    "    result = p.map(one_hot_enc,vector_input)\n",
    "    for each in result:\n",
    "        count, ex = each\n",
    "        existence.append(ex)\n",
    "        occurrence.append(count)\n",
    "\n",
    "    ex = pad_sequences(existence, maxlen=dic_size, padding='post', truncating='post')\n",
    "    count = pad_sequences(occurrence, maxlen=dic_size, padding='post', truncating='post', dtype=float)\n",
    "    return ex, count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Anaconda3-2021.05/lib/python3.8/site-packages/numpy/core/_asarray.py:171: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  return array(a, dtype, copy=False, order=order, subok=True)\n"
     ]
    }
   ],
   "source": [
    "summary_overall = []\n",
    "\n",
    "summary_count = []\n",
    "stop_words = set(stopwords.words('english'))\n",
    "ps = PorterStemmer()\n",
    "#word tokenization + eliminating stopwords + words stemming + eliminating short words\n",
    "def clean(string):\n",
    "    word_tokens = word_tokenize(string)\n",
    "    filtered_sentence = [ps.stem(w) for w in word_tokens if (not w in stop_words) and (len(w)>3)]\n",
    "    return filtered_sentence \n",
    "\n",
    "for lines in doc_list:\n",
    "    if lines is None:\n",
    "        continue\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "    clean_summary = []\n",
    "    sum_count = 0\n",
    "    for line in sent_tokenize(lines):\n",
    "        temp = clean(line)\n",
    "        if temp != []:\n",
    "            sum_count+= (len(temp))\n",
    "            for t in temp:\n",
    "                clean_summary.append(t)\n",
    "    summary_overall.append(clean_summary)\n",
    "    summary_count.append(sum_count)   \n",
    "    #print(summary_overall)\n",
    "    #print(summary_count)\n",
    "\n",
    "\n",
    "\n",
    "    '''if limited is True:\n",
    "        if len(title_overall)>= 500:\n",
    "            break'''\n",
    "\n",
    "\n",
    "## Store sentence vectors:\n",
    "#max_text_len = 20000\n",
    "#max_summary_len = 200\n",
    "#if only_summary is False:\n",
    "\n",
    "\n",
    "y_tokenizer = Tokenizer()\n",
    "y_tokenizer.fit_on_texts(summary_overall)\n",
    "\n",
    "sum_vec = y_tokenizer.texts_to_sequences(summary_overall)\n",
    "\n",
    "\n",
    "#y_train = pad_sequences(y_train, maxlen=max_summary_len, padding='post', truncating='post')\n",
    "#y_test = pad_sequences(y_train, maxlen=max_summary_len, padding='post', truncating='post')\n",
    "\n",
    "\n",
    "sum_voc_size = len(y_tokenizer.word_index)+1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## Save the preprocessed data to file\n",
    "np.savez('preprocessed_topic', summary_word2vec = sum_vec, labels = title_list,\n",
    "                             summary_voc_size=sum_voc_size, predict_label = label\n",
    "                        )\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[257, 197, 53, 109, 85, 62, 95, 136, 283, 187]\n",
      "651\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUc0lEQVR4nO3df2xdd3nH8fdDV0qpS5NSakLKMGxZR2mgUPNDKgKbUAi0I5W2FqoWBdYpmjQYiI4RQEJDm0SYVAQD/lgECA8KbgV0rYpgdAGLITEgoT/SUroWavqTeC1Ji0P5ke7ZH/cELm7se659r+3vPe+XZN1zzj3n+Hmuk0+++frceyIzkSSV63ErXYAkaWkMckkqnEEuSYUzyCWpcAa5JBXOIJekwhnkklQ4g1ySCmeQS30QEX+w0jWoOQxyrVoR8a6IuDcifh4Rt0XEpoj4dET8U9s+YxFxT9v6dES8MyJuioiDEfHJiBiOiK9U5/nPiFhb7TsSERkRb46IuyNif0T8dUS8sDr+QER8rO3cfxQRX4+IByPigYi4PCLWzPne74qIm4CDVR1fnNPTRyPiw3182dRABrlWpYg4FXgL8MLMPB54NTBd8/A/B84G/gT4M+ArwHuAk2j9mf/bOfu/GNgAvB74MPBe4JXAc4ALIuLlh8sCPgA8DXg28HTgH+ac60LgHGAN8Flg8+Gwr0bprwc+U7MPqRaDXKvVo8AxwGkRcXRmTmfmj2oe+9HM3JeZ9wL/BXwnM6/PzF8BVwHPn7P/P2bmLzPza8BB4POZOdN2/PMBMvOOzLwuM3+Vmf8LfAh4+Zxz/Utm3p2Zj2Tm/cA3gfOr5zYDD2Tmnq5eCakDg1yrUmbeAbyd1oh3JiImI+JpNQ/f17b8yBHWhxazf0ScXNVxb0Q8TGvEfdKcc909Z30CuLhavhhH4+oDg1yrVmZ+LjNfCjwDSOCDtEbMT2zb7anLWNIHqjqem5lPohXMMWefuR8n+u/AcyPidOBc4PJ+F6nmMci1KkXEqRHxiog4BvglrZHxo8ANwGsj4sSIeCqtUftyOR6YBQ5ExHrgnZ0OyMxfAl8APgd8NzPv6m+JaiKDXKvVMcAO4AHgp8DJtH5h+RngRlq/+PwacMUy1vR+4AXAQ8CXgS/VPG4C2IjTKuqT8MYSUn9FxB8CPwSempkPr3Q9GjyOyKU+iojHAe8AJg1x9YvvPpP6JCKOo3UFzE9oXXoo9YVTK5JUOKdWJKlwyzq1ctJJJ+XIyEjXxx08eJDjjjuu9wUVoKm9N7VvaG7vTe0bOve+Z8+eBzLzKfM9v6xBPjIywu7du7s+bmpqirGxsd4XVICm9t7UvqG5vTe1b+jce0T8ZKHjnVqRpMIZ5JJUOINckgpnkEtS4QxySSqcQS5JhTPIJalwBrkkFc4gl6TCDdynH45s/3Kt/aZ3nNPnSiRpeTgil6TCGeSSVDiDXJIKZ5BLUuEMckkqnEEuSYUzyCWpcB2vI4+IU4Er2jY9C3gf8G/V9hFgGrggM/f3vsT+qHu9OXjNuaTVreOIPDNvy8wzMvMM4EzgF8BVwHZgV2ZuAHZV65KkZdbt1Mom4EeZ+RNgCzBRbZ8AzuthXZKkmiIz6+8c8Sng+5n5sYg4kJlr2p7bn5lrj3DMNmAbwPDw8JmTk5NdFzk7O8vQ0FCtfffe+1DX5+9k4/oTen7OurrpfZA0tW9obu9N7Rs69z4+Pr4nM0fne752kEfE44H7gOdk5r66Qd5udHQ0d+/eXev7tevm7trdzH3XtZJz5E29s3hT+4bm9t7UvqFz7xGxYJB3M7XyGlqj8X3V+r6IWFd9k3XATBfnkiT1SDdBfiHw+bb1a4Ct1fJW4OpeFSVJqq9WkEfEE4GzgS+1bd4BnB0Rt1fP7eh9eZKkTmp9Hnlm/gJ48pxtD9K6ikWStIJ8Z6ckFc4gl6TCGeSSVDiDXJIKZ5BLUuEMckkqnEEuSYUzyCWpcAa5JBXOIJekwhnkklQ4g1ySCmeQS1LhDHJJKpxBLkmFM8glqXAGuSQVziCXpMIZ5JJUuFr37IyINcAngNOBBP4SuA24AhgBpoELMnN/P4pcaSPbv1xrv+kd5/S5Ekl6rLoj8o8AX83MPwWeB9wKbAd2ZeYGYFe1LklaZh2DPCKeBLwM+CRAZv46Mw8AW4CJarcJ4Lz+lChJWkhk5sI7RJwB7AR+QGs0vgd4G3BvZq5p229/Zq49wvHbgG0Aw8PDZ05OTnZd5OzsLENDQ7X23XvvQ12fv1c2rj+h5+fspvdB0tS+obm9N7Vv6Nz7+Pj4nswcne/5OkE+Cvw3cFZmficiPgI8DLy1TpC3Gx0dzd27dy/4/Y5kamqKsbGxWvvWnc/uh37MkXfT+yBpat/Q3N6b2jd07j0iFgzyOnPk9wD3ZOZ3qvUvAC8A9kXEuuqbrANm6hYtSeqdjletZOZPI+LuiDg1M28DNtGaZvkBsBXYUT1e3ddKB4hXwUjqpVqXHwJvBS6PiMcDPwbeTGs0f2VEXALcBZzfnxIlSQupFeSZeQNwpPmZTT2tRpLUNd/ZKUmFM8glqXAGuSQVziCXpMIZ5JJUuLqXH6qGlXxXqaTmckQuSYUzyCWpcAa5JBXOIJekwhnkklQ4g1ySCmeQS1LhDHJJKpxBLkmFM8glqXAGuSQVziCXpMIZ5JJUuFqffhgR08DPgUeBQ5k5GhEnAlcAI8A0cEFm7u9PmZKk+XQzIh/PzDMy8/BNmLcDuzJzA7CrWpckLbOlTK1sASaq5QngvCVXI0nqWmRm550i7gT2Awn8a2bujIgDmbmmbZ/9mbn2CMduA7YBDA8Pnzk5Odl1kbOzswwNDdXad++9D3V9/tVq4/oTuup9kDS1b2hu703tGzr3Pj4+vqdtNuQx6t4h6KzMvC8iTgaui4gf1i0wM3cCOwFGR0dzbGys7qG/NTU1Rd3j3jRAd+mZvmisq94HSVP7hub23tS+Yem915paycz7qscZ4CrgRcC+iFgHUD3OLLoKSdKidQzyiDguIo4/vAy8CrgZuAbYWu22Fbi6X0VKkuZXZ2plGLgqIg7v/7nM/GpEfA+4MiIuAe4Czu9fmZKk+XQM8sz8MfC8I2x/ENjUj6IkSfX5zk5JKpxBLkmFM8glqXAGuSQVziCXpMIZ5JJUOINckgpnkEtS4QxySSqcQS5JhTPIJalwBrkkFc4gl6TCGeSSVDiDXJIKZ5BLUuHq3nxZq9hIzRtOT+84p8+VSFoJjsglqXAGuSQVrnaQR8RREXF9RFxbrZ8YEddFxO3V49r+lSlJmk83I/K3Abe2rW8HdmXmBmBXtS5JWma1gjwiTgHOAT7RtnkLMFEtTwDn9bQySVItkZmdd4r4AvAB4Hjg7zLz3Ig4kJlr2vbZn5mPmV6JiG3ANoDh4eEzJycnuy5ydnaWoaGhWvvuvfehrs+/Wm1cf0Kt3uv2vHH9Cb0oa1l08zMfNE3tval9Q+fex8fH92Tm6HzPd7z8MCLOBWYyc09EjHVbYGbuBHYCjI6O5thY16dgamqKuse9qealeCWYvmisVu91e56+aOHzrCbd/MwHTVN7b2rfsPTe61xHfhbwuoh4LfAE4EkR8VlgX0Ssy8z7I2IdMLPoKiRJi9Zxjjwz352Zp2TmCPAG4OuZeTFwDbC12m0rcHXfqpQkzWsp15HvAM6OiNuBs6t1SdIy6+ot+pk5BUxVyw8Cm3pfkg4b2f5lLt14aKDm/SX1nu/slKTCGeSSVDiDXJIKZ5BLUuEMckkqXDE3lqh78wRJahpH5JJUOINckgpnkEtS4QxySSqcQS5JhTPIJalwBrkkFc4gl6TCGeSSVDiDXJIKZ5BLUuEMckkqnEEuSYXrGOQR8YSI+G5E3BgRt0TE+6vtJ0bEdRFxe/W4tv/lSpLmqjMi/xXwisx8HnAGsDkiXgJsB3Zl5gZgV7UuSVpmHYM8W2ar1aOrrwS2ABPV9gngvH4UKElaWGRm550ijgL2AH8MfDwz3xURBzJzTds++zPzMdMrEbEN2AYwPDx85uTkZNdFzs7OcudDj3Z93CAYPhb2PdKbc21cf0JvTrQMZmdnGRoaWukyVkRTe29q39C59/Hx8T2ZOTrf87XuEJSZjwJnRMQa4KqIOL1ugZm5E9gJMDo6mmNjY3UP/a2pqSku+9bBro8bBJduPMRle3tzI6fpi8Z6cp7lMDU1xWL+rAyCpvbe1L5h6b13ddVKZh4ApoDNwL6IWAdQPc4sugpJ0qJ1HOpFxFOA32TmgYg4Fngl8EHgGmArsKN6vLqfhWrp6t73dHrHOX2uRFIv1fk/+zpgoponfxxwZWZeGxHfBq6MiEuAu4Dz+1inJGkeHYM8M28Cnn+E7Q8Cm/pRlFZW3ZF7NxzlS/3jOzslqXAGuSQVziCXpMIZ5JJUOINckgpnkEtS4QxySSqcQS5JhTPIJalwBrkkFc4gl6TCGeSSVDiDXJIKZ5BLUuEMckkqnEEuSYXrzV19pQ68zZzUP47IJalwBrkkFa5jkEfE0yPiGxFxa0TcEhFvq7afGBHXRcTt1ePa/pcrSZqrzoj8EHBpZj4beAnwNxFxGrAd2JWZG4Bd1bokaZl1DPLMvD8zv18t/xy4FVgPbAEmqt0mgPP6VKMkaQGRmfV3jhgBvgmcDtyVmWvantufmY+ZXomIbcA2gOHh4TMnJye7LnJ2dpY7H3q06+MGwfCxsO+Rla5i+WxcfwLQ+pkPDQ2tcDUro6m9N7Vv6Nz7+Pj4nswcne/52pcfRsQQ8EXg7Zn5cETUOi4zdwI7AUZHR3NsbKzut/ytqakpLvvWwa6PGwSXbjzEZXubc5Xo9EVjQOtnvpg/K4Ogqb03tW9Yeu+1rlqJiKNphfjlmfmlavO+iFhXPb8OmFl0FZKkRatz1UoAnwRuzcwPtT11DbC1Wt4KXN378iRJndT5P/tZwBuBvRFxQ7XtPcAO4MqIuAS4Czi/LxVKkhbUMcgz81vAfBPim3pbjiSpW76zU5IKZ5BLUuEMckkqnEEuSYUzyCWpcAa5JBXOIJekwhnkklS45nwak4pw+N6el248xJsWuM+n9/aUfscRuSQVziCXpMIZ5JJUOINckgpnkEtS4QxySSqcQS5JhfM6chVpZIFrzNt5vbmawBG5JBXOIJekwnWcWomITwHnAjOZeXq17UTgCmAEmAYuyMz9/StT6q+6UzXdcFpHy6XOiPzTwOY527YDuzJzA7CrWpckrYCOQZ6Z3wR+NmfzFmCiWp4AzuttWZKkuiIzO+8UMQJc2za1ciAz17Q9vz8z185z7DZgG8Dw8PCZk5OTXRc5OzvLnQ892vVxg2D4WNj3yEpXsfya1PfG9Sf83vrs7CxDQ0MrVM3KaWrf0Ln38fHxPZk5Ot/zfb/8MDN3AjsBRkdHc2xsrOtzTE1Ncdm3Dva4sjJcuvEQl+1t3lWiTep7+qKx31ufmppiMX9PStfUvmHpvS/2qpV9EbEOoHqcWXQFkqQlWWyQXwNsrZa3Alf3phxJUrc6BnlEfB74NnBqRNwTEZcAO4CzI+J24OxqXZK0AjpOQmbmhfM8tanHtUiSFsF3dkpS4QxySSpcM67vkgaAn/io+Tgil6TCGeSSVDinViQtqJtPhnRaZ2U4IpekwhnkklQ4p1akAdOPm2RodXNELkmFM8glqXBOrUgrbO5UyKUbD/Emp0fUBUfkklQ4R+SSitf0jy9wRC5JhTPIJalwTq1IWnZHmgpZTb/kLW2qxhG5JBXOIJekwi1paiUiNgMfAY4CPpGZ3oRZarCmfTzAavlkyEWPyCPiKODjwGuA04ALI+K0XhUmSapnKVMrLwLuyMwfZ+avgUlgS2/KkiTVFZm5uAMj/gLYnJl/Va2/EXhxZr5lzn7bgG3V6qnAbYv4dicBDyyq0PI1tfem9g3N7b2pfUPn3p+RmU+Z78mlzJHHEbY95l+FzNwJ7FzC9yEidmfm6FLOUaqm9t7UvqG5vTe1b1h670uZWrkHeHrb+inAfUs4nyRpEZYS5N8DNkTEMyPi8cAbgGt6U5Ykqa5FT61k5qGIeAvwH7QuP/xUZt7Ss8p+35KmZgrX1N6b2jc0t/em9g1LnX5e7C87JUmrg+/slKTCGeSSVLhVHeQRsTkibouIOyJi+0rX02sR8amImImIm9u2nRgR10XE7dXj2rbn3l29FrdFxKtXpuqli4inR8Q3IuLWiLglIt5WbW9C70+IiO9GxI1V7++vtg9879B6R3hEXB8R11brTel7OiL2RsQNEbG72ta73jNzVX7R+gXqj4BnAY8HbgROW+m6etzjy4AXADe3bftnYHu1vB34YLV8WvUaHAM8s3ptjlrpHhbZ9zrgBdXy8cD/VP01ofcAhqrlo4HvAC9pQu9VP+8APgdcW603pe9p4KQ523rW+2oekQ/8RwBk5jeBn83ZvAWYqJYngPPatk9m5q8y807gDlqvUXEy8/7M/H61/HPgVmA9zeg9M3O2Wj26+koa0HtEnAKcA3yibfPA972AnvW+moN8PXB32/o91bZBN5yZ90Mr8ICTq+0D+XpExAjwfFoj00b0Xk0v3ADMANdlZlN6/zDw98D/tW1rQt/Q+sf6axGxp/rYEuhh76v5DkG1PgKgQQbu9YiIIeCLwNsz8+GII7XY2vUI24rtPTMfBc6IiDXAVRFx+gK7D0TvEXEuMJOZeyJirM4hR9hWXN9tzsrM+yLiZOC6iPjhAvt23ftqHpE39SMA9kXEOoDqcabaPlCvR0QcTSvEL8/ML1WbG9H7YZl5AJgCNjP4vZ8FvC4ipmlNk74iIj7L4PcNQGbeVz3OAFfRmirpWe+rOcib+hEA1wBbq+WtwNVt298QEcdExDOBDcB3V6C+JYvW0PuTwK2Z+aG2p5rQ+1OqkTgRcSzwSuCHDHjvmfnuzDwlM0do/V3+emZezID3DRARx0XE8YeXgVcBN9PL3lf6t7kdftP7WlpXNPwIeO9K19OH/j4P3A/8hta/wpcATwZ2AbdXjye27f/e6rW4DXjNSte/hL5fSuu/ijcBN1Rfr21I788Frq96vxl4X7V94Htv62eM3121MvB907ry7sbq65bDWdbL3n2LviQVbjVPrUiSajDIJalwBrkkFc4gl6TCGeSSVDiDXJIKZ5BLUuH+H1tCvzP4nlO8AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(summary_count[:10])\n",
    "\n",
    "print(len(summary_overall))\n",
    "overview = pd.DataFrame({'summary':summary_count})\n",
    "overview.hist(bins=30)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set([\"i\", \"me\", \"my\", \"myself\", \"we\", \"our\", \"ours\", \"ourselves\", \"you\", \"your\", \"yours\", \"yourself\", \"yourselves\", \"he\", \"him\", \"his\", \"himself\", \"she\", \"her\", \"hers\", \"herself\", \"it\", \"its\", \"itself\", \"they\", \"them\", \"their\", \"theirs\", \"themselves\", \"what\", \"which\", \"who\", \"whom\", \"this\", \"that\", \"these\", \"those\", \"am\", \"is\", \"are\", \"was\", \"were\", \"be\", \"been\", \"being\", \"have\", \"has\", \"had\", \"having\", \"do\", \"does\", \"did\", \"doing\", \"a\", \"an\", \"the\", \"and\", \"but\", \"if\", \"or\", \"because\", \"as\", \"until\", \"while\", \"of\", \"at\", \"by\", \"for\", \"with\", \"about\", \"against\", \"between\", \"into\", \"through\", \"during\", \"before\", \"after\", \"above\", \"below\", \"to\", \"from\", \"up\", \"down\", \"in\", \"out\", \"on\", \"off\", \"over\", \"under\", \"again\", \"further\", \"then\", \"once\", \"here\", \"there\", \"when\", \"where\", \"why\", \"how\", \"all\", \"any\", \"both\", \"each\", \"few\", \"more\", \"most\", \"other\", \"some\", \"such\", \"no\", \"nor\", \"not\", \"only\", \"own\", \"same\", \"so\", \"than\", \"too\", \"very\", \"s\", \"t\", \"can\", \"will\", \"just\", \"don\", \"should\", \"now\"]) \n",
    "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\",\n",
    "                           \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n",
    "                           \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",\n",
    "                           \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\",\n",
    "                           \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n",
    "                           \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\",\n",
    "                           \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\",\n",
    "                           \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\n",
    "                           \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\",\n",
    "                           \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n",
    "                           \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\",\n",
    "                           \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n",
    "                           \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\",\n",
    "                           \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\",\n",
    "                           \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\",\n",
    "                           \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n",
    "                           \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\",\n",
    "                           \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n",
    "                           \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\",\n",
    "                           \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\",\n",
    "                           \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\n",
    "                           \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\",\n",
    "                           \"you're\": \"you are\", \"you've\": \"you have\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    newString = text.lower()\n",
    "    newString = re.sub(r'\\([^)]*\\)', '', newString)\n",
    "    newString = re.sub('\"','', newString)\n",
    "    newString = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in newString.split(\" \")])    \n",
    "    newString = re.sub(r\"'s\\b\",\"\",newString)\n",
    "    newString = re.sub(\"[^a-zA-Z]\", \" \", newString) \n",
    "    newString = re.sub('[m]{2,}', 'mm', newString)\n",
    "    tokens = [w for w in newString.split() if not w in stop_words]\n",
    "    long_words=[]\n",
    "    for i in tokens:\n",
    "        if len(i)>1:                                          \n",
    "            long_words.append(i)   \n",
    "    return (\" \".join(long_words)).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_docs = []\n",
    "for doc in doc_list:\n",
    "    preprocessed_docs.append(preprocess(doc)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_model = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(docs):\n",
    "    vec_list = list()\n",
    "    for doc in docs:\n",
    "        doc_model = spacy_model(doc)\n",
    "        doc_vec = np.zeros(doc_model[0].vector.shape)\n",
    "        for tok in doc_model:\n",
    "            if tok.text not in stop_words:\n",
    "                doc_vec += tok.vector\n",
    "        vec_list.append(doc_vec)\n",
    "    return vec_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec_list = vectorize(preprocessed_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cluster and plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxaklEQVR4nO2df5gU5ZXvP2eGxmlQGfwVxhm4QFRWkZ+yhgTjk5+DcRJlVUDX3JirG5/d692g7pIMymMwS9ZJ2KyaJzfZR+NuMBoFCTuoo0Ii2Ztogi4EhGBCQsCVGXBljUOyMEAz894/umvo6a6qruqu6q7qPp/nmWe6366qPt0w3/etc857jhhjUBRFUWqLukoboCiKopQfFX9FUZQaRMVfURSlBlHxVxRFqUFU/BVFUWqQYZU2wCtnnXWWGT9+fKXNUBRFiRVbtmz5L2PM2bnjsRH/8ePHs3nz5kqboSiKEitE5D/sxtXtoyiKUoOo+CuKotQgKv6Koig1SGx8/oqiKJUglUrR3d3N0aNHK22KKw0NDbS0tJBIJDwdr+KvKIriQnd3N6eddhrjx49HRCptji3GGN555x26u7uZMGGCp3NU/BVFqQqWb1rOU795igEzQJ3UMf+C+SydvbTk6x49ejTSwg8gIpx55pkcPHjQ8zkq/oqixJ7lm5azateqwecDZmDweRATQJSF38KvjRrwVRQl9jz1m6d8jSsq/oqiVAEDZsDXeBx54YUXmDRpEueddx4dHR0lX0/FX1GU2FMn9lLmNB43+vv7ue2223j++ed5/fXXeeKJJ3j99ddLumZ1fDOKotQ08y+Y72s8TDq39jCnYyMT2ruY07GRzq09JV/z1Vdf5bzzzmPixIkMHz6c66+/nnXr1pV0TRV/RVFiz9LZS1k4aeHgSr9O6lg4aWEgwV4/dG7tYcnaHfT09mGAnt4+lqzdUfIE0NPTw9ixYweft7S00NNT2jU120dRlKpg6eylZRf7XFas30Vfqn/IWF+qnxXrdzFvRnPR17XrtV5qBpKu/BVFUQJif2+fr3GvtLS0sG/fvsHn3d3dnHvuuSVdU8VfUZSapGtPF61rWpm6ciqta1rp2tNV8jXPbUz6GvfKn/7pn/Lb3/6WvXv3cvz4cZ588kmuuuqqkq6p4q8oSs3RtaeLZT9bxoHDBzAYDhw+wLKfLSt5Alg8dxLJRP2QsWSinsVzJ5V03WHDhvHNb36TuXPncuGFF7JgwQImT55c2jVLOltRFCWGPPiLBznaP7RQ29H+ozz4iwdpm9hW9HUtv/6K9bvY39vHuY1JFs+dVJK/3+LKK6/kyiuvLPk6Fir+iqLUHG8dfsvXuB/mzWgOROzDRt0+iqLUHGNGjvE1Xo2o+CuKUnMsmrmIhvqGIWMN9Q0smrmoQhaVH3X7KIpSc1h+/Qd/8SBvHX6LMSPHsGjmopL8/XFDxV9RlJqkbWJbTYl9Lur2URRFqUFU/BVFUSLOzTffzDnnnMPFF18c2DVV/BVFUSLOZz/7WV544YVAr6niryiKEiTbV8P9F8OyxvTv7atLvuTll1/OGWecUbptWWjAV1GUyNG1pyuemTjbV8Mzn4dUppDboX3p5wBTF1TOLht05a8oSqQIq+5OWXjxyyeF3yLVlx6PGIGJv4jUi8hWEXk28/wMEfmhiPw283t01rFLRGS3iOwSkblB2aAoSvxxq7sTeQ51+xuvIEGu/BcBv8p63g68aIw5H3gx8xwRuQi4HpgMXAF8S0TqURRFIdy6O6EzqsXfeAUJRPxFpAVoA76TNXw1sDLzeCUwL2v8SWPMMWPMXmA3cGkQdiiKEn9iXXfno/dAIqd2fyKZHi+BG264gfe///3s2rWLlpYWHnnkkZKuB8EFfB8AvgCcljX2HmPMAQBjzAEROScz3gxsyjquOzOWh4jcCtwKMG7cuIBMVRQlyiyauYhlP1s2xPUTm7o7VlD3xS+nXT2jWtLCX2Kw94knngjAuKGULP4i8kngbWPMFhH5kJdTbMbyG1QCxpiHgIcAZs2aZXuMoijVRezr7kxdELnMHjuCWPnPAa4SkSuBBuB0EXkM+E8Racqs+puAtzPHdwNjs85vAfYHYIeiKFVCrdfdKQcl+/yNMUuMMS3GmPGkA7kbjTGfBp4GbsocdhOwLvP4aeB6ETlFRCYA5wOvlmqHoiiK4p0wN3l1AKtF5BbgTWA+gDFmp4isBl4HTgC3GWP6Q7RDURRFySFQ8TfG/Bvwb5nH7wAfdTjuK8BXgnxvRVEUxTu6w1dRFKUGUfFXFEWJOPv27ePDH/4wF154IZMnT+bBB0vf7ayF3RRFUSLOsGHD+PrXv87MmTP54x//yCWXXMLHP/5xLrrooqKvqSt/RVGUAOna00XrmlamrpxK65rWQArSNTU1MXPmTABOO+00LrzwQnp6ekq6pq78FUVRAsKqSGrtTrYqkgKB7Vt444032Lp1K+973/tKuo6u/BVFUQIi7Iqk//3f/821117LAw88wOmnn17StVT8FUVRAiLMiqSpVIprr72WG2+8kWuuuabk66n4K4qiBERYFUmNMdxyyy1ceOGF3HnnnSVdy0LFX1EUJSAWzVxEQ33DkLEgKpK+/PLLfO9732Pjxo1Mnz6d6dOn89xzz5V0TQ34KoqiBERYFUkvu+wyjAm2sLGKv6IoSoDEpSKpun0URVFqEBV/RVGUAgTtcgkDvzaq20dRqoCuPV3x7XwVcRoaGnjnnXc488wzERF6j/Xy9uG3SQ2kSNQlOGfkOTSe0lhRG40xvPPOOzQ0NBQ+OIOKv6LEnHLsKq1lWlpa6O7u5uDBg/Sd6KP3aC8mq/NsN900NjSSHJZ0uUr4NDQ00NLS4vl4FX9FiTluu0pV/EsnkUgwYcIEAFrXtHLg8IG8Y5pGNrHhug3lNq0k1OevKDEnzF2lylCq6btW8VeUmBPWrlIln2r6rlX8FSXmhLWrVMmnmr5r9fkrSswJa1epkk81fdcSh/xVgFmzZpnNmzdX2gxFUZRYISJbjDGzcsfV7aMoilKDqPgriqLUICr+iqIoNYiKv6IoSg2i4q8oilKDqPgrSo3QtaeL1jWtTF05ldY1rXTt6aq0SUoF0Tx/RakBtPibkouu/BWlROKwonYr/qbUJrryV5QSiMuKOqyCZNpHIL7oyl9RSiAuK+owCpJZE9+BwwcwmMGJL8g7nzjcVcUVFX9FKYG4lPgNoyBZ2BNfOSaXWkbdPkpF6Nzaw73P7OTdIykAGpMJll01mXkzmitsmT/GjBxj29wjaiV+wyhIFvbEp01qwqVk8ReRscCjwBhgAHjIGPOgiJwBrALGA28AC4wx72bOWQLcAvQDnzfGrC/VDiU+dG7tYfGa10j1nywq2NuXYvFTrwFUdALo3NrDivW72N/bx7mNSRbPneRqz6KZi4b4/CG6JX7bJrYFKpphT3xxuauKK0G4fU4Af2OMuRCYDdwmIhcB7cCLxpjzgRczz8m8dj0wGbgC+JaI1AdghxITVqzfNUT4LVIDhhXrd1XAojSdW3tYsnYHPb19GKCnt48la3fQubXH8Zy2iW0s+8AymkY2IQhNI5tY9oFlNbEyDbu2fTU1TokiJa/8jTEHgAOZx38UkV8BzcDVwIcyh60E/g34Ymb8SWPMMWCviOwGLgV+XqotSjzY39tX1Gths2L9LvpS/UPG+lL9rFi/y3X1H/SKOi6EXds+TndVcSRQn7+IjAdmAK8A78lMDBhjDojIOZnDmoFNWad1Z8bsrncrcCvAuHHjgjRVqSDnNibpcRD5cxuTZbbmJE4TTyUnpKgT5sRXTY1Tokhg4i8ipwI/AG43xvxBRBwPtRmz7ShjjHkIeAjSzVyCsFOpPIvnTsrz+QMk6oTFcydVyCrnScnrhOQ3511z5AtTq3dV5SCQVE8RSZAW/seNMWszw/8pIk2Z15uAtzPj3cDYrNNbgP1B2KHEg3kzmllx3TRGj0gMjjUmE6yYP62iwd7FcyeRTAwNPyUT9Z4mJL9piZrGqFSakts4SnqJvxL4vTHm9qzxFcA7xpgOEWkHzjDGfEFEJgPfJ+3nP5d0MPh8Y0x//tVPom0clSBxyurxm+1j0bqm1TbzpWlkExuu21Dy8WGyfNNynvrNUwyYAeqkjvkXzGfp7KVltUEJD6c2jkG4feYA/xPYISLbMmN3AR3AahG5BXgTmA9gjNkpIquB10lnCt1WSPiV+FKsmIZ5XSurxwruWlk9kL4rKcY+v2mJUUljXL5pOat2rRp8PmAGBp/rBFDdaAN3JTRyRRbSbpT7rplS0gRQ6nXndGy09e3XizBgTFGTVBRX/l5iCtMencaAGcg7t07qeO0zrwVih1JZtIG7UnbcUiedWNq5g/cueY7x7V28d8lzLO3cEch1s3HK3uk3xnN+fy5+c97DzpG3iym0/7SdDz75wSFxBTvhdxtXqgcVfyU0/KZOLu3cwWOb3qQ/czfabwyPbXqTGx8eugWk1JRML9k7fiYT8L/ZK+zNYXalEQB6j/UOCSzXib0EOI0r1YPW9lFCw2/q5BOv7LMdf/l3v6dza8+gG8bpunUiTGjvKui2WTx3Up7byA6/+f1+0xLDTGN0ix1k18eZf8H8IT5/i/kXzA/FLiU66PSueKJzaw9zOjYyob2LOR0bPblE/KROdm7tGVzx25G9Cre7Lnh328yb0cx910yhuTGJAE5bUiq54axUCpVAsCaHpbOXsnDSwsGVfp3UsXDSQg321gAa8FUKsrRzB49venPITjyvAVYvWTl2AVw7BAavAQxet07EduJobkzycvtHCtrntOEsrH0H5djcldtkJpdKpJQqlUEDvkpRdG7tyRN+8O4TnzejmZfbP8L9C6cDcMeqbXl3DnYBXDuyV/UAL7d/hL0dbQw4LGC8uG2cisyd2jAsNOEvx+YuK6YwaviovNe0Po4CKv5KAVas32Vfe4Oh4urmFipULdOvbz134nFyz3hx2zi9d2+mz0DQlLPzV9vENl664SU6PthRk1VHFXc04FvjFHLLuAmzJa6FNk0VqpbpVujNiWy77AK4XssyFApKB+2iqcTmLq2Po9ihK/8axFqlj2/v4o5V21zr1zutngWG+N7d8u4LpWY6BXCt97Ej267cAG5zY9Lzhi+3oHQYLhqtUa9EBV351xi5q3Q3X/6K9bvo6e1Dco4T4MbZ4wbF1Unce3r7mNOxkVHJBL19+W6UbAFvSNTZ+v1N5v1yg825q/piyzJY52Tf/Xz4T85mxfpd9J7ZQd3w4NoIdu3pou9E/nelPnilEqj41xhegqvWHUD2BGEJcLONa8jNbdPT20eiXkjUCamBkxJuCbhTtk021vsGXR/IInviyJ4cTx3Ta3t8MS4ap+ybUcNHseR9S9Qto5QdFf8aw0twtV4kb4KwBNgudbLQpqlUv2H0iAQjhg/LE/AZX97gKvyQLvdcKGUzKLInR5NqRIb35h1TjIvGacftiMQIFX6lIqj4VxmFAriFgqvJRL2jiDtNHNb1lz2909a9A+nsma33tOaNv+shq8a5L1A+pVYRzf6Mxw7OpaFpLVJ30sZiXTRRqeKpKBYa8K0i7FIq71i1bUhxNLsAp6WtVqC0ucjUycPHTzi+Vifiq1BaNu8eSXnaVby0c0fBAHYhsj/jiT/M4OiBaxg43giGktIkNdCrRA0V/yrCzp9vgMc2vTk4Adhlxty/cDpvdLTxcvtHmDej2TH75vCxE45C6rRZyqLfGFshbkwmHM4YijWRjXcoL1HqZjSL3M9+4g8zGHjzbv5u2gtsuG6DrfB37emidU0rU1dOpXVNq202UNhVPCuFl89ezcT586vbp4pw8+c/vulNAH7864ODLpH7F063dYlYY/c+s3OIW6a3LzUkf9/re1v0pfq5fdU2lj29E5H0it6PS8cS9tx9BOB9M1oh7LJ/3FxHuYFcKx0UGDJRVGMzcq+fvVqJ++fX2j5xYPtqePHLcKgbRrXAR++BqQvyDpt+7wZHn7sddhk8ls+80Kar5kxKpDWZONXXCZPsAPSE9i5H8beODTpLCKLVjrHc1PJnh/h8/jDbOCphsn01PPN5SGXE+NC+9HPImwBS/f4acOSupDf/x+/5wZYeT3V2enr7eCxzNwGUXfghvaK3JqtC7253t+CHz63/HJve2jT4fPaY2Tw89+GaDuTW8meH+H9+9flHnRe/fFL4LVJ96fEsOrf2cPh48a2Q+1L9PLbpTU/CHxUaRyQGA9xe6Ev18zerX/NVlhryhR9g01ub+Nz6z9V0ILeWPzvE//Or+EedQ90Fx60sn6jgw41fNMlEPcbge7IqplVjrvBnjwcZyI1b8LBag9heifvnV/GPOokR9uPJ0YMP731mZ6RW7NaO4DC575opHHKIb1hZTIXwmwlkR1DtGMtV6jlIwm5FGXXi/vk14Ftpnr0TtnwXTD9IPVzyWfjkP6Zf274a1n7O/rzkGfCJr3Lk+XtoOHKA/eYsvnZiAU8PXFYuy31TH2BQ+IGF0x0D01Zw10uDmOzjnWIBU1ZOcTx3x03B3HHFJXioxA9t5hJFnr0TNj+SFn5I/978CKy8Kv08x68/hL7fw7rbGNF3gDqBlrr/4h8SD3FV3Uvh210khYT/gYXTSdR5u2dYsnYHH/6Tsx0rcubuZ6h3ySkt5AKaPWa2r/FiiHLwMG7uKMUbKv6VZMt37cf3/r/0xODk77foPz7k6XA5wf2Jb0d6AnBj3oxmVsyf5mnjV1+qnx//+mDejmQrqLu0c8dgF7G9HW18fcE0x7LR1nlOLqCH5z6cJ/RWtk9QRDV4GEd3lOINFf9ys3013H8xLGs8ueK3Y/MjRV2+Xgwdie/EcgKY07ERgG1fyq8BZMf+3j7mzWjmw39y9pDxfmOG7GqGoTub3a7nxMNzH2bHTTsGf4IUfohu8LCcnceU8qLiHxSDoj4K7j0j/fv+i9Pj2cc88/l0rn7BzHQ8HpPPCDnOF4atLnygDz49exzvOW247/P8BH57evu4fdU2pt/rzcfdOCJ9h/DEK/tsX88dt+4Eiq1dFCZRDR5G2R2llIZu8gqC7auh83/DQCb7xFrR527IssvZD4lz5Z1Ar/fEK29SoPKyLXs72pjTsdFXm0avu5StEIJTLMFpvJS2j37x0wYyiu0Wx4wcYxuIrrQ7SikdXfl7IdtVk72a374avjohnZEz4CBYqT54/ovpx4V8+AHyrhkZ6PWKEX5Il5ywuoEFzaG+lGuevlOQt5S2j+A9AFoN/vKouqOU0tFUz0LkllcASCRh2p/DLx51Fv08cpsRhssJU8edqb+MdOpnqYwekeBoasAxnfPTs8exfJ5zmmYx2HXkGibDOHX4qRw6dmjI6r5a0jeDbmKvlBenVE8V/0Lcf3HGRx8/ugfO4rLj36i0GUVj+ebtXEYCjr2BIRzhB+d8/Gwa6htY9oFlLPnpEozDhC+ICqlSFjTPvxi2r46t8EPwfv9yc+T4CdtcfquBvNMOX0iXri62eYwbXgKdVjaMm188rm4gpXpQ8bcj25cfY3oJ1u8fNsnE0P+O7x5J8YMtPVx7SXNe85nl86a4ZucU08XLC14DnW8dfsvWX56Lpk0qlUKzfSwGa+bvo9z++bCIiUdvkKOp/JLU1mauYhrH96X6ufeZnYHW8F80c1Gez9+OMSPH5DVwcXIBadqkUgkqtvIXkStEZJeI7BaR9krZAZxM1Rx08cRMNR0YLYcrbYIvnL51pzRRLxu33j3inhHkl7aJbVx93tWux2Rnw7RNbGPDdRvYftN2mkY22R6vaZNKJaiI+ItIPfB/gU8AFwE3iMhFlbAFSKdies7aiQ/7zZmBXStRJ9Tn1N1J1Aufnj1uUHzd6ueUilP9/UIbt4CSK3dm07Wni3W71zm+Pmr4KMfNWZo2qUSJSq38LwV2G2P2GGOOA08C7supMOn7fcXeOiyMga+dyG/1WAzNjUlWzJ/G1+dPGyL0qX7Dj399cDAo67Vi5xsdbZ5KLmdTyIfvtkHLTw/fQtiVO7BYOGkhL93wkusmriju4lVqk0qJfzOQnUbTnRkrD8/emd6wtWxU+qdKCSLHf+TwdKbNHau2sWL9rjyh7+nt43GfHcA6t/bk1ePxglvxtXkzmh0LwgVZtsHNP79u97qCmTttE9tYNHMRY0aO4a3Db9HxageXPXGZVsxUyk6lxN/OP5C3bBSRW0Vks4hsPnjwYOnvun01/N3ZmaJp1eHXt8MY+OnA5JKvU18nHD8xQE9v32D3Kzuh9/tNLlm7g2dfc8+Vd8JtFb/sqsmOJZ6Dws0/7yVzJ3fXb++xXg4dPxRK6qeWYlbcqJT4dwNjs563APtzDzLGPGSMmWWMmXX22f5XikPYvhrW3ZZXBrnasIR4r7EPLvqhf8CQGhgq7X6E3rpryKUv1e+5fk8ubqv4Uss2FKJrTxd9J9xdSIUyd9zcRhBc6mc1lJbIRSezYKmU+P87cL6ITBCR4cD1wNOBv8uzd5507az9XNULP6RvqUTgM/U/4t5h/1xRW/5sZnGi25hM8MDC6b5W8Z1be5jTsZE7Vm0D4P6F03m5/SOBCv+yny2j91iv63GFMne8pHUeOHzAUdy8CmC1lWKuxsms0lRE/I0xJ4D/A6wHfgWsNsbsDPRNrC5ZNYoI3Fi/0fPx9SIFO14NXtvjNX/864OO13O7xqG+VF4aZ73IoM8/N+hrNbDPdk8FvcGr0IodvGXueE3rtBM3PwJYbaWYq20yiwIVy/M3xjxnjLnAGPNeY8xXAn+DLf8S+CXjRj35m6acGDCGvR1tDBTI2Ekm6rkxK73TjZ7ePscMILd3sVw782Y0s3jupLwAc66wr1i/Ky8OEURz9mzc6vn4ydzxk9aZK25+BLBcncHK5YqptsksClRveQfjXfiqlX4f/7yW4Lr51C0f+vJ5U3i5/SO2rplSyXXteBF2pyBwUCmeboLWNLKJ7TdtZ8N1GzylbLZNbEN8FLjOFjc/AuhUWuLylss9v3chyumKiWqbyzhTveJf4xgDj/fnl0SwI1Eng4JrrbRzXx89IsH+3r4hbhfLNTN6ROGeu17JDdB6EXanCSuoFE8310IxG7ScyjzYkS1ufgTQaSeyl3RUr5TTFaMb5IKnesU/Ea+iZkFhgH4jPNr/Mb504mZP55zaMGxQcHMzZhqTCZB0mQQ7f/q8Gc1svafVU9N1KwPH6djmxmRegNaLsNtNWEGmeLq5FqzVvh/3h1OZh1xyxc2vAP6k+yd5Y0GKczldMbpBLniqV/w/9QD+OshWBwLUieEz9T/ipeGf99TI/d0jqSHlE6ySCXs72hh5yjBSOW287PzpbuWVLc5tTLJ47iRf+fhehD3sFE+nFbcl4n7cH117ujiSOpI33lDfwMJJC13Fza8Ahi3O5XbFZNdJ8upmU5yp7mYu21en6/ZUYfkGrxwxw2lP/YWn3b7JRP2gaHZu7WHF+l2uvXebM2I+b0az5z691ntA2p+/v7dvcFJwEutsW+pF6DdmyHuHjV33Lqthi5+OXXbXAWg8pZH2S9sDF7OwO4kV+l6UaODUzKW6SzpPXZD+yebeM042WLcjMRJSR6iWHcAj5DhfGLaap48XFv/sFb1bqWQLywUEhcsr576Hnxx867js62e/d+51lnbu4IlX9tFvDPUi3PC+sSV19bKE7L5X7uPQ8UMANAw76X7xusJ2ShdNDku7sFrXtAbaKtGu/HSQfvLcktXamSxeVLf423HJZ53z/xPJjLuIqrpj8NPRqycT1PVaqydbzJ/a/CYv/67wd7a/t29wNe9l5Q/uWT/Z5y3t3MFjm94cfN5vzODzUts6Hus/Nvi491gvy362DEi7OexW2LnuD6dJwnITWSJtPQdKEtJyiHPbxDYV+5hS3W4fJ569E7Z8d+gdwKix8NF7ht4pbF8d+25eAD3mLNrqvs2hvtSg0Dq5dIppY2O1VcwWXTcakwmOnRjaeN3O5ZQ9MdyxaputXQLs7TgpPu9d8pzt3oJ6EX5335U+P9lJ3FwoTivsXPeH0zXqpI4Bm9TkcjV61wbt1U1tun2c+OQ/pn/csDp7lZNRY+GOX5YUqzAMDXMfMcP5amoBI08fxrYvtQJpH/qR4yccz7f86l45tzHJE69463WcTNQjgu0q/vZV21j29E4OHz8xGGS23DuNIxK8eyQ/qJybDeRkt5/PY4eba8frCtupC5id8Lu9Z5Dk+u2DuutQok/1ZvuUwvbV8Mzny9y8XdJ3HpC++/jiXrjmYRB/m6gEOGHqGDBC98BZg8FeKy/eKoVgJ6QWfoVy8dxJrufkZuH0urx3b1/KNrvo3SOpvNwtuwwhp3ISpTaaKZTZ4iUTxcrWGTXcWxnxcmxg0rIJtYuKvx0vfhlSwTUAKYzArJvzg9NTFxS1U7kOw8Rjj3PZ8W8MZvlYK2Q//nyLxmTCcSPX6BEJ5s1odhVdK23UCvIWu/kq+67GKZ3zhveNzTvPbdwrTjtm3Yqw2dE2sY0RiREFjyvXBiYtm1C7qPjbcajb+bVRYwFJ/77mYUie4f/6kvW1J8+Aax5ydkONanG7kO3oAYa2b8xeIRdT8kDEvhl8MlHPlz6V7hvgR3RL2XxlSAu/U7bQ8nlT+PTscYOTUb2kW016DfY6bdbKzrHPxW9ZAzdhLfcGJi2bULvUZsC3EPdfbO/ysXzy2SxrpKS00EQSpv05/HZD+j2lPh2ItgLQkHZB5d6JJM+AyX8Gr31/6GuJJP8+5V5uf/1820war/n4hRg9IsGXPjU5L9PGa4rl9Hs3FF3TPzfIGxRe89ZLzZ93C/z+/WV/X1Zfu+bqVz9OAV8Vfzssn3+OqPKpb+S7ZpwmCkvEPeGQY2O9J6RdUYe603cC2VlJVmDa7jUbOrf2sHjNa3l+9USdcGrDMNdYQDbW6rtYrNiDXxeU9d5WxpLXVNHc97Y716uoT1051bY+jyBsv2l7wfd32uwFlRFezfapblT8/eJVVJ0miml/nr8qLwa7u40ScVp126VgOuF19e2Wz2+3c7cQyUQ9117SzA+29DimihayJ3fSsc69Z/snPIl6EDtnu/Z0cddLd1U0xVOpDZzEX33+TkxdkBbdZb3p306r6akL0qvz7FjAp76R9uHnjhcTH3CLPxSJUx2eQ32pITVyRo9IOFZH8hK0LdRkxaoh9EZHG7+778qCJaJHj0hw3zVT+PGvDxZdv99ts5hX/3cQFSbbJrbhtPDSYKtSDmozzz9o7MpI2I3b3SUU2lblGvAtjnMbk7Z+/1HJdOaOtdFqydodtpZ5rZjpdVeuhTVm3Q3kfjPvHklx7zM7HV1TXoLZbiWiv+mxHEJQO2e97gxWlDDQlX85sbtLmHVz2k1kRyJ5MugbIIvnTiJRl7+mP3z8xOCq3CkltF6koHvF6qfrFFh2E2nrbqC5MWk78djl+1t4uRtxKxHtp2qmnwqTThlEWqNeqSS68i83dncJ42Zn4gs22T4uwdtimTej2XYFneo3g6tyJ4EeMKag8BcK5HoRabcJwsr3z54cvN6N2BWgyz436Fo1XnbQarBVqQQq/lHAyW20fXUmm8hbJo8fnHbZWqLr5BoqJNyFNpF5FWmn97ew8v39Zvtku5aKyRTyi9sOWmui8Sv2mp2jBIGKf1TJjQ8c2gdrb00XmgvgrqCQuBdaITvhtmL3U4O/UInoUlJNrbhGOXCr5Dl15VTf4q21eJSgUJ9/VLEtMZFxdBzal54Ytq8u+vJuHbI6t/Zw7zM7hwhvYzLhKZXS6c7AbVeuHVZ3LruWj3aTkBVnmNDeNaQrWaVxC94W0/S8nLV4/LSmVOKHin9UKZTimeorqeqoU+tDgMVrXsuLBxw+Zl8FNJcg++nOm9HMti+18sDC6a4tGgullGZT7knCqSZQNn7Eu1y1ePy0plTiibp9osqolsJVRUvcA2Dn/pjTsTFv9y9AasA4pmjmXhOC9akXctN4TSnNDUa7dQMrBjdfvDVut4kMvIt3udJDC8UqlPij4h9VPnqPfU2fIZh0QDjAQLCbz97pNbtdvKWUfvCLW+5+Nn73HfihkC/eEkyn3cFexTvs1owWWu2z+lG3T1QZsicAnCp4BuH/z8Ytm8fuNT8ul7Bwy93PxuskUQz3vXKfJ198qbn9fvYilIJW+6x+VPyjzGCJiUPpss+DE0EOqT74178MZAJYPHcSifr8iSZRJ7Z+e7fVdLnwGmfwOkn4pWtP12Bj91xyV8pBiLefDWbFohvQqh91+8QFay+AUwlp05++A7COLRLL/ZG9CawxmWDZVZNtXSNhrqa94jXOUGz6aiHcgrV2K+U4ND1324AWh30GcbCx0qj4xw23QLCVAVSi/99PHnyxm8GCxovNhSaJYgXDzQ8e55Wy3SQVh30GcbAxCqj4x41CgeBD3b5r/JdCWKvpsHCaJEoRDKcMnMZTGgMXm0qvaJ2ygO566S4gGuKqmUreUJ9/3LACwU6N3ZOjs5rPm8ADwrk47RcIawdtWHn6pWyecvKPt1/aHohtFlHIvXe6yxkwA5HZB6CZSt5Q8Y8jUxfAn/1TfjVQ63nuXUGJG8IKYVXizG7SHgZhZha5lWEotLu1XBk45dzd64Rbtk+5bXFCM5W8oeIfV5yayPS9a398CE1hyoW12r991bbQMovchMHLCrscGThRWNEW2rEchdW1Zip5Q8U/zth1G3Nq/hJCU5hykL3adyKIzKJCohaFVW0UVrTWXU6d2EtHFFbX5boTizslBXxFZAXwKeA48DvgfxljejOvLQFuAfqBzxtj1mfGLwG+CySB54BFJi6NhOOAXUA4pKYw5aBQiWgIJrMoO7XRLngL5V/V5gZ3L2+5nHW714W+u7cQ1ndVjp3GxRKHdNpKU+rK/4fAxcaYqcBvgCUAInIRcD0wGbgC+JbIYITy28CtwPmZnytKtEHJxskdFFK2T9gUWtUHmVlkuW6aRjbZvl7OVa1dcHfd7nVcfd7VkVjR6uo6/pS08jfGbMh6ugm4LvP4auBJY8wxYK+I7AYuFZE3gNONMT8HEJFHgXnA86XYoeTg1Bwmhrg1dfHTH8AP5aqf44ZTcPcn3T9hw3UbHM4qL7q6jjdB+vxv5qSINwPZO5G6M2PNmce547aIyK0isllENh88eDBAU5W4YFe6wamHb1BEYVUbheCuUt0UXPmLyI8Au/vdu40x6zLH3A2cAB63TrM53riM22KMeQh4CGDWrFkaF6hBsnfl9vT2DendG3RJ5mwqvaotV+lmpXYpuPI3xnzMGHOxzY8l/DcBnwRuzArcdgPZVchagP2Z8RabcUVxxNpH0NyYzFsplLuIXLnQdEUlbEpy+4jIFcAXgauMMUeyXnoauF5EThGRCaQDu68aYw4AfxSR2SIiwGeAdaXYoNQOUSgiVy6i4HpSqptSa/t8EzgF+GFay9lkjPlLY8xOEVkNvE7aHXSbMcbK1/srTqZ6Po8GexWPRKWIXLmotOtJqW5KzfY5z+W1rwBfsRnfDFxcyvsqtUncisgpzlS6QJ2iVT2VGBFGf2Cl/IRZclknFe9IXDbXzpo1y2zevLnSZiiKUiJOfYybRjaVtIchd1KBdJC81mMlIrLFGDMrd1xr+yhKDOja00XrmlamrpxasMpo1AlrD0MUqp7GCRV/RYk4QdTxj9LkEVaBOt0Y5w8Vf0WJOKWuaKPQBCabsPYwRKHqaZxQ8VeUiFPqijZq7pCw9jDoxjh/aLaPokScUks9RNEdEsYehuyy3JrtUxgVf0WJOKVWGQ2rTlAU0yqraWNc2N+vun0UJeKU6iYJwx1iF0do/2k7yzctL/qayknKEafRPH9FqQGCXkU65eoDdHywA1D3SykEuRfCKc9f3T6KUgME7Q5xixd0vNrB0RNHQ9nBWyuUI06jbh+l5ujc2sOcjo1MaO9iTsdGOrf2VNqk2OEWL+g91hup7KI4Uo60VRV/pabo3NrDkrU76Ontw3CyIYxOAP4oJl5w4PCBSGwyiwPlSFtV8VdqihXrdw2pCgrxaQizfNNypj06jSkrpzDt0WkVDa62TWxj4aSFeeMN9Q2MGj7K8bwobDKLA+Xo56ABX6WmmNDeZds3VIC9HdH1Ry/ftJxVu1bljS+ctJCls5dWwKI0doFkIC811Q6vwcsoppTGCQ34KgrxbQjz1G+echyvpPi7BZItwTYObbq9BC/DLP9c66jbR6kpFs+dRDJRP2QsDg1hBsyAr/FK0zaxjQ3XbWD7TdtpGtlke4yX4GXUSlNUEyr+Sk0xb0Yz910zhebGJAI0Nya575opkW8IUyf2f6pO41GilOCl014Cp/GoE6Xqqur2UWqOeTOaIy/2ucy/YL6tz3/+BfMrYI0/Sqm5Uyd1tnc3cZj0comaC0vFX1FigOXXf+o3TzFgBqiTOuZfML+i/n4/FLvJLG7uLjfcXFgq/oqiOLJ09tI8sa/2TJimkU2OZQ7iRtSqq8bv3klRFCB6TVrCoJpq9Eet2YyKv6LElFrIhCnHZqdyEbWJTN0+ihJTouZGCItqqdEftWYzKv6KElPCatKihEeUJjJ1+yhKTImaG0GJF7ryV5SYEjU3ghIvVPwVJcZEyY2gxAt1+yiKotQgKv6Koig1iIq/oiiKR6JUmK1U1OevKIrigagVZisVXfkriqJ4oNp2VAci/iLytyJiROSsrLElIrJbRHaJyNys8UtEZEfmtW+IiARhg6IoSphU247qksVfRMYCHwfezBq7CLgemAxcAXxLRKz2Sd8GbgXOz/xcUaoNiqIoYRO1wmylEsTK/37gCzCkUefVwJPGmGPGmL3AbuBSEWkCTjfG/NykO8c/CswLwAZFUZRQqbYd1SUFfEXkKqDHGPNajvemGdiU9bw7M5bKPM4dd7r+raTvEhg3blwppiqKopREte2oLij+IvIjwO6+5m7gLqDV7jSbMeMybosx5iHgIYBZs2Y5HqcoilIOqmlHdUHxN8Z8zG5cRKYAEwBr1d8C/EJELiW9oh+bdXgLsD8z3mIzriiKopSRon3+xpgdxphzjDHjjTHjSQv7TGPMW8DTwPUicoqITCAd2H3VGHMA+KOIzM5k+XwGWFf6x1AUJcpU0+aoaiGUTV7GmJ0ishp4HTgB3GaM6c+8/FfAd4Ek8HzmR1GUKqXaNkdVC5JOuok+s2bNMps3b660GYqi+KR1TatjE/YN122ogEW1hYhsMcbMyh3XHb6KooRKtW2OqhZU/BVFCZVq2xxVLaj4K4oSKtW2Oapa0KqeiqKESrVtjqoWVPwVRQmdatocVS2o20dRFKUGUfFXFEWpQVT8FUVRahAVf0VRlBpExV9RFKUGiU15BxE5CPxH1tBZwH9VyBwvRNm+KNsG0bYvyrZBtO2Lsm0QbftKse1/GGPOzh2MjfjnIiKb7epVRIUo2xdl2yDa9kXZNoi2fVG2DaJtXxi2qdtHURSlBlHxVxRFqUHiLP4PVdqAAkTZvijbBtG2L8q2QbTti7JtEG37Arcttj5/RVEUpXjivPJXFEVRikTFX1EUpQaJjfiLyN+KiBGRs7LGlojIbhHZJSJzs8YvEZEdmde+kWkWH5ZdK0Tk1yKyXUT+VUQao2Sfjb1XZOzZLSLt5XrfrPcfKyI/FpFfichOEVmUGT9DRH4oIr/N/B6ddY7t9xiijfUislVEno2gbY0isibzf+5XIvL+qNgnIndk/k1/KSJPiEhDJW0TkX8WkbdF5JdZY77tCevv1cG+8umJMSbyP8BYYD3pTV5nZcYuAl4DTgEmAL8D6jOvvQq8HxDSDeI/EaJtrcCwzOOvAl+Nkn05ttZn7JgIDM/Yd1GZ/y2bgJmZx6cBv8l8V18D2jPj7V6+xxBtvBP4PvBs5nmUbFsJ/EXm8XCgMQr2Ac3AXiCZeb4a+GwlbQMuB2YCv8wa821PWH+vDvaVTU/isvK/H/gCkB2dvhp40hhzzBizF9gNXCoiTcDpxpifm/Q38ygwLyzDjDEbjDEnMk83AS1Rsi+HS4Hdxpg9xpjjwJMZO8uGMeaAMeYXmcd/BH5FWjiuJi1sZH7Pyzy2/R7Dsk9EWoA24DtZw1Gx7XTSgvEIgDHmuDGmNyr2ke4PkhSRYcAIYH8lbTPG/AT4fc6wL3vC/Hu1s6+cehJ58ReRq4AeY8xrOS81A/uynndnxpozj3PHy8HNpGdeiKZ9TjZVBBEZD8wAXgHeY4w5AOkJAjgnc1i5bX6A9EJjIGssKrZNBA4C/5JxS31HREZGwT5jTA/wD8CbwAHgkDFmQxRsy8GvPVWrJ5Ho5CUiPwLsujnfDdxF+lYo7zSbMeMyXjRu9hlj1mWOuRs4ATxebvt8UMn3HoKInAr8ALjdGPMHFzdl2WwWkU8CbxtjtojIh7ycYjMW5vc5jLSb4K+NMa+IyIOkXRdOlPO7G016dToB6AWeEpFPu51iM1bJvPNI/b2WQ08iIf7GmI/ZjYvIFNL/mV7LiEML8AsRuZT0DDc26/AW0reZ3Zy8VcoeD9y+LDtvAj4JfDRz60U57fOBk01lRUQSpIX/cWPM2szwf4pIkzHmQOZW9u3MeDltngNcJSJXAg3A6SLyWERss96v2xjzSub5GtLiHwX7PgbsNcYcBBCRtcAHImJbNn7tKfvfa9n0JMgAS9g/wBucDPhOZmgAZA8nAyD/DszmZADkyhBtugJ4HTg7ZzwS9uXYNCxjxwROBnwnl/nfUEj7JR/IGV/B0EDc1wp9jyHb+SFOBnwjYxvwU2BS5vGyjG0Vtw94H7CTtK9fSPvT/7rStgHjGRpQ9W1PmH+vNvaVTU9C/QMK4R/yDTLin3l+N+mo9y6yItzALOCXmde+SWYnc0g27Sbti9uW+fmnKNlnY++VpDNsfkfabVXuf8PLSN+Wbs/6zq4EzgReBH6b+X1Goe8xZDs/xEnxj4xtwHRgc+b76wRGR8U+4F7g15n/29/LCFXFbAOeIB1/SJFeId9SjD1h/b062Fc2PdHyDoqiKDVI5LN9FEVRlOBR8VcURalBVPwVRVFqEBV/RVGUGkTFX1EUpQZR8VcURalBVPwVRVFqkP8PnWu1PLf58ccAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# reduce to two dimensions using pca\n",
    "pca = PCA(2)\n",
    "reduced_vec_list = pca.fit_transform(vec_list)\n",
    "\n",
    "# perform kmeans on the reduced data\n",
    "kmeans = KMeans(n_clusters= 3)\n",
    "label = kmeans.fit_predict(reduced_vec_list)\n",
    "\n",
    "# plot clusters\n",
    "u_labels = np.unique(label)\n",
    "for i in u_labels:\n",
    "    \n",
    "    plt.scatter(reduced_vec_list[label == i , 0] , reduced_vec_list[label == i , 1] , label = i)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print(type(reduced_vec_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez(\"labels\", label=label_list)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "40d3a090f54c6569ab1632332b64b2c03c39dcf918b08424e98f38b5ae0af88f"
  },
  "kernelspec": {
   "display_name": "CSE204",
   "language": "python",
   "name": "cse204"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}