utils/TheGuardian.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('the_guardian_articles.csv', nrows=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Federal investigators are examining the first ...</td>\n",
       "      <td>Dan Tynan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple is reportedly considering a potential ta...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>From turning your photos into gifs to riding b...</td>\n",
       "      <td>Stuart Dredge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It’s Friday – what a week!</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>V\\nideo games are often criticised for their o...</td>\n",
       "      <td>Keith Stuart</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text         author\n",
       "0  Federal investigators are examining the first ...      Dan Tynan\n",
       "1  Apple is reportedly considering a potential ta...            NaN\n",
       "2  From turning your photos into gifs to riding b...  Stuart Dredge\n",
       "3                         It’s Friday – what a week!            NaN\n",
       "4  V\\nideo games are often criticised for their o...   Keith Stuart"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp=spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['doc']=df.text.apply(lambda x: nlp(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "length = df['doc'].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['len']=length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     100.000000\n",
       "mean      859.190000\n",
       "std       412.826613\n",
       "min         3.000000\n",
       "25%       517.750000\n",
       "50%       844.000000\n",
       "75%      1087.500000\n",
       "max      2172.000000\n",
       "Name: len, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.len.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "      <th>doc</th>\n",
       "      <th>len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It’s Friday – what a week!</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(It, ’s, Friday, –, what, a, week, !)</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Chatterbox: Thursday</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(Chatterbox, :, Thursday)</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          text author                                    doc  \\\n",
       "3   It’s Friday – what a week!    NaN  (It, ’s, Friday, –, what, a, week, !)   \n",
       "61        Chatterbox: Thursday    NaN              (Chatterbox, :, Thursday)   \n",
       "\n",
       "    len  \n",
       "3     8  \n",
       "61    3  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['len']<50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ents']=df['doc'].apply(lambda doc: doc.ents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ents']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc=df['doc'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ORDINAL',\n",
       " 'PRODUCT',\n",
       " 'PERSON',\n",
       " 'CARDINAL',\n",
       " 'GPE',\n",
       " 'PRODUCT',\n",
       " 'DATE',\n",
       " 'ORG',\n",
       " 'GPE',\n",
       " 'PERSON',\n",
       " 'PERSON',\n",
       " 'PERSON',\n",
       " 'PRODUCT',\n",
       " 'ORG',\n",
       " 'DATE',\n",
       " 'PRODUCT',\n",
       " 'ORG',\n",
       " 'PRODUCT',\n",
       " 'PRODUCT',\n",
       " 'PERSON',\n",
       " 'DATE',\n",
       " 'PERSON',\n",
       " 'GPE',\n",
       " 'PERSON',\n",
       " 'CARDINAL',\n",
       " 'PRODUCT',\n",
       " 'QUANTITY',\n",
       " 'QUANTITY',\n",
       " 'ORG',\n",
       " 'PERSON',\n",
       " 'DATE',\n",
       " 'PRODUCT',\n",
       " 'DATE',\n",
       " 'PERSON',\n",
       " 'ORG',\n",
       " 'PERSON',\n",
       " 'DATE',\n",
       " 'PRODUCT']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ent.label_ for ent in doc.ents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_ents=[ent for ents in df['ents'] for ent in ents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ent in all_ents:\n",
    "    ent.merge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CARDINAL',\n",
       " 'DATE',\n",
       " 'EVENT',\n",
       " 'FAC',\n",
       " 'GPE',\n",
       " 'LANGUAGE',\n",
       " 'LAW',\n",
       " 'LOC',\n",
       " 'MONEY',\n",
       " 'NORP',\n",
       " 'ORDINAL',\n",
       " 'ORG',\n",
       " 'PERCENT',\n",
       " 'PERSON',\n",
       " 'PRODUCT',\n",
       " 'QUANTITY',\n",
       " 'TIME',\n",
       " 'WORK_OF_ART'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{ent.label_ for ent in all_ents}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_freq={label: 0 for label in {ent.label_ for ent in all_ents}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ent in all_ents:\n",
    "    labels_freq[ent.label_]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CARDINAL': 448,\n",
       " 'DATE': 676,\n",
       " 'EVENT': 14,\n",
       " 'FAC': 83,\n",
       " 'GPE': 750,\n",
       " 'LANGUAGE': 6,\n",
       " 'LAW': 10,\n",
       " 'LOC': 62,\n",
       " 'MONEY': 132,\n",
       " 'NORP': 148,\n",
       " 'ORDINAL': 138,\n",
       " 'ORG': 1241,\n",
       " 'PERCENT': 58,\n",
       " 'PERSON': 947,\n",
       " 'PRODUCT': 190,\n",
       " 'QUANTITY': 52,\n",
       " 'TIME': 57,\n",
       " 'WORK_OF_ART': 50}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_persons=[ent for ent in all_ents if ent.label_=='PERSON']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[That complacency has been shattered by the revolt of marginalised citizens around the world, angry at “elite” failings and looking to the likes of Donald Trump.,\n",
       " When Mark Zuckerberg was asked by his own employees if they should act to prevent certain Donald Trump stories gaining traction his answer was absolutely not.]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[person.sent for person in all_persons if 'Trump' in person.text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_art=[ent.text for ent in all_ents if ent.label_=='WORK_OF_ART']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Android 1.5 –',\n",
       " 'Ansip and Jourova',\n",
       " 'Brought',\n",
       " 'Corporations Can Make Doing Good an Integral',\n",
       " 'Encrypting Your Laptop',\n",
       " 'Film Now',\n",
       " 'Future Shock',\n",
       " 'Game Boy',\n",
       " 'Get Windows 10',\n",
       " 'Guitar Archive',\n",
       " 'Harry Potter DVD',\n",
       " 'Marvel’s Iron Man',\n",
       " 'Most Corrupt Candidate',\n",
       " 'Musk told Fortune.\\n\\n',\n",
       " 'Nobel',\n",
       " 'Paid for by”',\n",
       " 'Playboy',\n",
       " 'Pokémon',\n",
       " 'Pokémon Go',\n",
       " 'Queers Hate Techies',\n",
       " 'Radar',\n",
       " 'Scaling at GitHub’ - with a large portion of the problem being the',\n",
       " 'Space Pavilion',\n",
       " 'Star Wars',\n",
       " 'The Associated Press',\n",
       " 'The Center',\n",
       " 'The Dyson Demo',\n",
       " 'The Infinite Monkey Cage',\n",
       " 'The Managed Heart',\n",
       " 'The Secret Tesla Motors Master Plan',\n",
       " 'The “My Eyes Only”',\n",
       " 'The “Sorry',\n",
       " 'They Live',\n",
       " 'Totem Adornments',\n",
       " 'Wi-Fried',\n",
       " 'Your Autopilot Has Arrived',\n",
       " 'a Tesla Model S',\n",
       " 'the Gold Coast',\n",
       " 'the Guinness Book of Records',\n",
       " 'the Live from Jodrell Bank',\n",
       " 'the Play Store',\n",
       " '“Gotta Catch ‘Em All!',\n",
       " '“Killing it',\n",
       " '“Light Detection and Ranging”',\n",
       " '“See More Settings',\n",
       " '“Train',\n",
       " '“Upgrade now”',\n",
       " '”\\n\\n',\n",
       " '” Adam Summers'}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(all_art)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ent_filter(ents, kind):\n",
    "    return [ent for ent in ents if ent.label_==kind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['persons']=df['ents'].apply(lambda ents: ent_filter(ents, 'PERSON'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "      <th>doc</th>\n",
       "      <th>len</th>\n",
       "      <th>ents</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>persons</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Federal investigators are examining the first ...</td>\n",
       "      <td>Dan Tynan</td>\n",
       "      <td>(Federal, investigators, are, examining, the, ...</td>\n",
       "      <td>445</td>\n",
       "      <td>((first), (Tesla), (Joshua Brown), (40-year), ...</td>\n",
       "      <td>-0.047798</td>\n",
       "      <td>[(Joshua Brown), (Brown), (Tesla), (Brown), (E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple is reportedly considering a potential ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(Apple, is, reportedly, considering, a, potent...</td>\n",
       "      <td>300</td>\n",
       "      <td>((Apple), (Jay Z’s), (Tidal), (Tidal), (Kanye ...</td>\n",
       "      <td>0.128217</td>\n",
       "      <td>[(Jay Z’s), (Tidal), (Tidal), (Kanye West), (M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>From turning your photos into gifs to riding b...</td>\n",
       "      <td>Stuart Dredge</td>\n",
       "      <td>(From, turning, your, photos, into, gifs, to, ...</td>\n",
       "      <td>1511</td>\n",
       "      <td>((this month), (IAP), (Android), (monthly), (t...</td>\n",
       "      <td>0.196480</td>\n",
       "      <td>[(Toca Boca), (Netflix), (Rodeo Stampede), (Ka...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It’s Friday – what a week!</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(It, ’s, Friday, –, what, a week, !)</td>\n",
       "      <td>8</td>\n",
       "      <td>((Friday), (a week))</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>V\\nideo games are often criticised for their o...</td>\n",
       "      <td>Keith Stuart</td>\n",
       "      <td>(V, \\n, ideo, games, are, often, criticised, f...</td>\n",
       "      <td>775</td>\n",
       "      <td>((British), (this week), (last minute), (Ubiso...</td>\n",
       "      <td>0.085346</td>\n",
       "      <td>[(Borgias), (Creed II), (Machiavelli), (Sid Me...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text         author  \\\n",
       "0  Federal investigators are examining the first ...      Dan Tynan   \n",
       "1  Apple is reportedly considering a potential ta...            NaN   \n",
       "2  From turning your photos into gifs to riding b...  Stuart Dredge   \n",
       "3                         It’s Friday – what a week!            NaN   \n",
       "4  V\\nideo games are often criticised for their o...   Keith Stuart   \n",
       "\n",
       "                                                 doc   len  \\\n",
       "0  (Federal, investigators, are, examining, the, ...   445   \n",
       "1  (Apple, is, reportedly, considering, a, potent...   300   \n",
       "2  (From, turning, your, photos, into, gifs, to, ...  1511   \n",
       "3               (It, ’s, Friday, –, what, a week, !)     8   \n",
       "4  (V, \\n, ideo, games, are, often, criticised, f...   775   \n",
       "\n",
       "                                                ents  sentiment  \\\n",
       "0  ((first), (Tesla), (Joshua Brown), (40-year), ...  -0.047798   \n",
       "1  ((Apple), (Jay Z’s), (Tidal), (Tidal), (Kanye ...   0.128217   \n",
       "2  ((this month), (IAP), (Android), (monthly), (t...   0.196480   \n",
       "3                               ((Friday), (a week))   0.000000   \n",
       "4  ((British), (this week), (last minute), (Ubiso...   0.085346   \n",
       "\n",
       "                                             persons  \n",
       "0  [(Joshua Brown), (Brown), (Tesla), (Brown), (E...  \n",
       "1  [(Jay Z’s), (Tidal), (Tidal), (Kanye West), (M...  \n",
       "2  [(Toca Boca), (Netflix), (Rodeo Stampede), (Ka...  \n",
       "3                                                 []  \n",
       "4  [(Borgias), (Creed II), (Machiavelli), (Sid Me...  "
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob.sentiments import PatternAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "analyzer=PatternAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.04779785431959346"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyzer.analyze(doc.text).polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc.sentiment=analyzer.analyze(doc.text).polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.17277777194976807"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in df['doc']:\n",
    "    doc.sentiment=analyzer.analyze(doc.text).polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sentiment']=df['doc'].apply(lambda d: d.sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>author</th>\n",
       "      <th>doc</th>\n",
       "      <th>len</th>\n",
       "      <th>ents</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>persons</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Federal investigators are examining the first ...</td>\n",
       "      <td>Dan Tynan</td>\n",
       "      <td>(Federal, investigators, are, examining, the, ...</td>\n",
       "      <td>445</td>\n",
       "      <td>((first), (Tesla), (Joshua Brown), (40-year), ...</td>\n",
       "      <td>-0.047798</td>\n",
       "      <td>[(Joshua Brown), (Brown), (Tesla), (Brown), (E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Apple is reportedly considering a potential ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(Apple, is, reportedly, considering, a, potent...</td>\n",
       "      <td>300</td>\n",
       "      <td>((Apple), (Jay Z’s), (Tidal), (Tidal), (Kanye ...</td>\n",
       "      <td>0.128217</td>\n",
       "      <td>[(Jay Z’s), (Tidal), (Tidal), (Kanye West), (M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>From turning your photos into gifs to riding b...</td>\n",
       "      <td>Stuart Dredge</td>\n",
       "      <td>(From, turning, your, photos, into, gifs, to, ...</td>\n",
       "      <td>1511</td>\n",
       "      <td>((this month), (IAP), (Android), (monthly), (t...</td>\n",
       "      <td>0.196480</td>\n",
       "      <td>[(Toca Boca), (Netflix), (Rodeo Stampede), (Ka...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It’s Friday – what a week!</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(It, ’s, Friday, –, what, a week, !)</td>\n",
       "      <td>8</td>\n",
       "      <td>((Friday), (a week))</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>V\\nideo games are often criticised for their o...</td>\n",
       "      <td>Keith Stuart</td>\n",
       "      <td>(V, \\n, ideo, games, are, often, criticised, f...</td>\n",
       "      <td>775</td>\n",
       "      <td>((British), (this week), (last minute), (Ubiso...</td>\n",
       "      <td>0.085346</td>\n",
       "      <td>[(Borgias), (Creed II), (Machiavelli), (Sid Me...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text         author  \\\n",
       "0  Federal investigators are examining the first ...      Dan Tynan   \n",
       "1  Apple is reportedly considering a potential ta...            NaN   \n",
       "2  From turning your photos into gifs to riding b...  Stuart Dredge   \n",
       "3                         It’s Friday – what a week!            NaN   \n",
       "4  V\\nideo games are often criticised for their o...   Keith Stuart   \n",
       "\n",
       "                                                 doc   len  \\\n",
       "0  (Federal, investigators, are, examining, the, ...   445   \n",
       "1  (Apple, is, reportedly, considering, a, potent...   300   \n",
       "2  (From, turning, your, photos, into, gifs, to, ...  1511   \n",
       "3               (It, ’s, Friday, –, what, a week, !)     8   \n",
       "4  (V, \\n, ideo, games, are, often, criticised, f...   775   \n",
       "\n",
       "                                                ents  sentiment  \\\n",
       "0  ((first), (Tesla), (Joshua Brown), (40-year), ...  -0.047798   \n",
       "1  ((Apple), (Jay Z’s), (Tidal), (Tidal), (Kanye ...   0.128217   \n",
       "2  ((this month), (IAP), (Android), (monthly), (t...   0.196480   \n",
       "3                               ((Friday), (a week))   0.000000   \n",
       "4  ((British), (this week), (last minute), (Ubiso...   0.085346   \n",
       "\n",
       "                                             persons  \n",
       "0  [(Joshua Brown), (Brown), (Tesla), (Brown), (E...  \n",
       "1  [(Jay Z’s), (Tidal), (Tidal), (Kanye West), (M...  \n",
       "2  [(Toca Boca), (Netflix), (Rodeo Stampede), (Ka...  \n",
       "3                                                 []  \n",
       "4  [(Borgias), (Creed II), (Machiavelli), (Sid Me...  "
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "jay=df['persons'][1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "jay_token=df['persons'][1][0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities=[]\n",
    "for i, person in enumerate(all_persons):\n",
    "    similarities.append((person, jay_token.similarity(person[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(Jay Z’s, 1.0),\n",
       " (Jay Z’s, 1.0),\n",
       " (Joshua Brown, 0.76865244),\n",
       " (Hillary Clinton, 0.75971967),\n",
       " (Mark Twain, 0.7441862),\n",
       " (Banks, 0.7419728),\n",
       " (John Lewis, 0.7366898),\n",
       " (Joshua Brown, 0.7233147),\n",
       " (Taylor Swift, 0.7210378),\n",
       " (Avid Life’s, 0.71220845),\n",
       " (Jonathan Ross’s, 0.6951377),\n",
       " (Harry Potter, 0.6825176),\n",
       " (Madonna, 0.6817763),\n",
       " (Levy, 0.67637736),\n",
       " (Joshua Brown, 0.6725146),\n",
       " (Charles, 0.6723214),\n",
       " (Alvin Toffler, 0.66949624),\n",
       " (Creed II, 0.66866565),\n",
       " (Rob Segal, 0.6670843),\n",
       " (Harmony Korine, 0.6658535),\n",
       " (Aiden Byrne, 0.65130115),\n",
       " (Alex Gibney, 0.65062433),\n",
       " (Gibney, 0.64995867),\n",
       " (Sebastien Missoffe, 0.64696395),\n",
       " (Trak, 0.64406985),\n",
       " (Tim Cook, 0.63841516),\n",
       " (Tim Cook, 0.6363678),\n",
       " (Ray Lakeland, 0.63124907),\n",
       " (Toffler, 0.62472504),\n",
       " (Arthur Conan Doyle, 0.6245247),\n",
       " (Eddie Waring, 0.6232251),\n",
       " (Ukip, 0.6231637),\n",
       " (Google Photos, 0.6219252),\n",
       " (Elon Musk, 0.6197937),\n",
       " (Gwyneth Paltrow, 0.61873734),\n",
       " (Neil Harbisson, 0.6132424),\n",
       " (this Quantified Self, 0.6043565),\n",
       " (Lovell, 0.6012998),\n",
       " (Joshua Brown, 0.60090894),\n",
       " (Wolfgang Münchau, 0.59876955),\n",
       " (Brown, 0.5981765),\n",
       " (Jafar Panahi, 0.5973827),\n",
       " (Elizabeth Holmes, 0.5964909),\n",
       " (Maria Miller, 0.5929319),\n",
       " (Avid Life Media, 0.589575),\n",
       " (Tidal, 0.58733237),\n",
       " (Elon Musk’s, 0.58567846),\n",
       " (Google DeepMind, 0.58239126),\n",
       " (Spencer Behrend, 0.57758546),\n",
       " (Maryanne Demasi, 0.5740511),\n",
       " (Prius, 0.5634026),\n",
       " (Ukip, 0.56307983),\n",
       " (Nougat, 0.5579957),\n",
       " (Nougat, 0.5575726),\n",
       " (Churchill, 0.55505544),\n",
       " (Martin Kettle, 0.55461085),\n",
       " (Tesla, 0.5541797),\n",
       " (Pokémon Go, 0.552374),\n",
       " (Victoria Climbié, 0.5521505),\n",
       " (Melanie Wilkinson, 0.5450526),\n",
       " (Pokémon Trainer’s, 0.5415974),\n",
       " (Bryant Walker Smith, 0.5412579),\n",
       " (Warby Parker, 0.5400297),\n",
       " (Bloomberg, 0.5398797),\n",
       " (Maguire, 0.5360662),\n",
       " (Ukip, 0.53053766),\n",
       " (Tesla, 0.5279397),\n",
       " (Epsom Derby, 0.522079),\n",
       " (Jay Z, 0.5173008),\n",
       " (Brown, 0.51692885),\n",
       " (Edward Snowden, 0.5092622),\n",
       " (Steve Wozniak, 0.4978214),\n",
       " (Bad Babysitter, 0.48657072),\n",
       " (Sean Allen, 0.47892097),\n",
       " (Elon Musk, 0.47642097),\n",
       " (Avid Life Media, 0.47493237),\n",
       " (AutoDesk, 0.47069886),\n",
       " (Bluedot, 0.46888342),\n",
       " (Ion, 0.46777686),\n",
       " (Wai Ming Wong, 0.46054125),\n",
       " (Churchill, 0.45556068),\n",
       " (Autopilot, 0.45545048),\n",
       " (Farm Heroes Saga, 0.45072758),\n",
       " (Android – the, 0.4493652),\n",
       " (Arron Banks, 0.4419597),\n",
       " (Lovell, 0.43472588),\n",
       " (Trak, 0.4262912),\n",
       " (Robert Masse, 0.4240291),\n",
       " (George Freeman, 0.42353576),\n",
       " (Vaughan Bell, 0.4228209),\n",
       " (Phonebloks, 0.42134696),\n",
       " (Tidal, 0.42100126),\n",
       " (AI, 0.41414085),\n",
       " (Overwatch, 0.41404596),\n",
       " (Tim Pettersen, 0.41038212),\n",
       " (Taxi Tehran, 0.4101468),\n",
       " (Evgeny Naidenov, 0.40810087),\n",
       " (ARMED GUARDS, 0.40103364),\n",
       " (Iris Milano, 0.40018505),\n",
       " (Toffler, 0.3985066),\n",
       " (Barney, 0.39736354),\n",
       " (Moto, 0.3763874),\n",
       " (Phillips, 0.37030822),\n",
       " (Maureen Tricario, 0.3687827),\n",
       " (Warby Parker, 0.3600211),\n",
       " (Carlos Trinidad, 0.3599424),\n",
       " (Miller, 0.3428365),\n",
       " (Carl Sagan, 0.3254397),\n",
       " (Reid, 0.3249028),\n",
       " (Taylor Swift, 0.3172228),\n",
       " (McCoy, 0.31548873),\n",
       " (Steve\n",
       "  , 0.31087744),\n",
       " (Aiden Byrne, 0.30128908),\n",
       " (Kat Borlongan, 0.2975629),\n",
       " (Earthquakes, 0.293371),\n",
       " (Bitcoin, 0.2931406),\n",
       " (Hares, 0.292183),\n",
       " (Murdoch, 0.29047886),\n",
       " (Amazon Fresh, 0.28471476),\n",
       " (Eileen, 0.28439155),\n",
       " (Goddard Gunster, 0.282577),\n",
       " (Diamond Reynolds, 0.28121033),\n",
       " (Brown, 0.2798879),\n",
       " (Tesla, 0.2797746),\n",
       " (Autopilot, 0.27510867),\n",
       " (Robert Shields, 0.26502228),\n",
       " (John Mackey, 0.26146978),\n",
       " (Merlin, 0.2599436),\n",
       " (Brandt, 0.25612974),\n",
       " (mm, 0.25224125),\n",
       " (Citrus Zinger, 0.25112697),\n",
       " (Johnson, 0.24727255),\n",
       " (Tim, 0.24330008),\n",
       " (companies’, 0.24315085),\n",
       " (Ashley Madison, 0.24096878),\n",
       " (Kiva, 0.2406523),\n",
       " (Muriel, 0.2405789),\n",
       " (Kurt, 0.2404707),\n",
       " (Louis, 0.2396439),\n",
       " (Matthew, 0.23918702),\n",
       " (Brown, 0.23542677),\n",
       " (Holmes, 0.23309833),\n",
       " (Harry Potter, 0.22914013),\n",
       " (Kez’ Brown, 0.22852333),\n",
       " (Paul Dacre, 0.22804962),\n",
       " (Greg Sommer, 0.22604139),\n",
       " (George Brock, 0.22552522),\n",
       " (James Bond, 0.22527874),\n",
       " (Shaun Aimoto, 0.22036561),\n",
       " (Tesla, 0.21822996),\n",
       " (Joshua Brown’s, 0.21816742),\n",
       " (Kurt Coleman, 0.21778174),\n",
       " (Joshua Brown, 0.21694101),\n",
       " (Blair, 0.21637641),\n",
       " (Brown, 0.21285996),\n",
       " (Derren Brown’s, 0.21269324),\n",
       " (Rzevski, 0.21236457),\n",
       " (Hares, 0.21231882),\n",
       " (Aubrey, 0.21149373),\n",
       " (John Chen, 0.21149324),\n",
       " (Bluedot, 0.21070993),\n",
       " (Daniel Zeichner, 0.20933558),\n",
       " (Som Pastor, 0.20822616),\n",
       " (Level 39, 0.20562072),\n",
       " (Oliver Ebel, 0.2049012),\n",
       " (Zappp, 0.20484747),\n",
       " (Elon Musk, 0.20457914),\n",
       " (Tidal, 0.20457745),\n",
       " (Sridhar Lakshmanan, 0.20422055),\n",
       " (Rob Moffat, 0.20400539),\n",
       " (John Baruch, 0.20070498),\n",
       " (Vince Zampella, 0.20056523),\n",
       " (Jobs, 0.19987746),\n",
       " (Earth Wars, 0.1984387),\n",
       " (Flea, 0.19797218),\n",
       " (Mark Zuckerberg, 0.19365937),\n",
       " (George, 0.19332892),\n",
       " (Richard Finlayson, 0.19192107),\n",
       " (Donald Trump, 0.19079743),\n",
       " (Froyo, 0.19077757),\n",
       " (Pedal, 0.18950714),\n",
       " (Linux Kernel Development, 0.18921018),\n",
       " (John Lewis, 0.18874495),\n",
       " (Jeff Bezos, 0.1887056),\n",
       " (Sid Meier, 0.18847486),\n",
       " (Philip Green, 0.1852103),\n",
       " (Doug Gurr, 0.18439405),\n",
       " (Brown, 0.18437615),\n",
       " (Joshua Brown, 0.18337503),\n",
       " (Musk, 0.18285182),\n",
       " (Bryant, 0.18276003),\n",
       " (Warren, 0.1820423),\n",
       " (Kurt Coleman, 0.18203188),\n",
       " (Huskisson, 0.18154216),\n",
       " (Email Lucy, 0.18120915),\n",
       " (AI, 0.18119489),\n",
       " (Ashley Madison, 0.18082914),\n",
       " (Parker, 0.17967308),\n",
       " (Spotify, 0.17680086),\n",
       " (Fiona, 0.1759251),\n",
       " (Respawn, 0.17534736),\n",
       " (Tasmina Ahmed-Sheikh, 0.17533606),\n",
       " (Steve, 0.17517051),\n",
       " (Mary “Missy” Cummings, 0.17494765),\n",
       " (Elon Musk, 0.17439824),\n",
       " (Matthew Garrett, 0.17432211),\n",
       " (Tesla, 0.17384234),\n",
       " (Caldicott, 0.17317648),\n",
       " (William Lakeland, 0.17266327),\n",
       " (Ashley Madison, 0.17174903),\n",
       " (Brown, 0.17084287),\n",
       " (Check Point, 0.1695769),\n",
       " (Garrett, 0.16892682),\n",
       " (Autopilot, 0.1680504),\n",
       " (Obama, 0.16790393),\n",
       " (Martin, 0.16781867),\n",
       " (Ghost Train, 0.16670898),\n",
       " (Parker, 0.16658309),\n",
       " (Arlie Hochschild, 0.16654435),\n",
       " (Tidal, 0.16620761),\n",
       " (Miller, 0.16603822),\n",
       " (Spotify, 0.16565844),\n",
       " (Model X, 0.16548587),\n",
       " (Peaches, 0.16492124),\n",
       " (Musk, 0.16468035),\n",
       " (Dickinson, 0.16461457),\n",
       " (William Huskisson, 0.16444924),\n",
       " (\n",
       "  \n",
       "  \n",
       "  , 0.16343258),\n",
       " (Alli, 0.16290343),\n",
       " (Brown, 0.16288868),\n",
       " (Vladimir Nabokov, 0.16285169),\n",
       " (Narendra Modi, 0.16206256),\n",
       " (Boris Johnson, 0.16178258),\n",
       " (Charlotte Holloway, 0.16161002),\n",
       " (Tidal, 0.16146818),\n",
       " (Deng Xiaoping’s, 0.1611076),\n",
       " (Pete Lawrence, 0.16086629),\n",
       " (Gibney, 0.16004732),\n",
       " (From Karim, 0.159534),\n",
       " (Rob Rickell, 0.15928434),\n",
       " (Macs, 0.15922986),\n",
       " (Gerard Grech, 0.158864),\n",
       " (Model X, 0.15876241),\n",
       " (Wiley, 0.15764052),\n",
       " (Suikoden II, 0.1575493),\n",
       " (Gibney, 0.15601577),\n",
       " (Corbyn, 0.15458871),\n",
       " (Twitter, 0.15319891),\n",
       " (Tony Stark, 0.15294914),\n",
       " (Talulah Riley, 0.15272039),\n",
       " (Bloom, 0.15268098),\n",
       " (Heidi Farrell, 0.15263157),\n",
       " (Garner, 0.15182783),\n",
       " (Tim Burgess, 0.15045798),\n",
       " (Mario Kart, 0.15035921),\n",
       " (Drew McCoy, 0.15023144),\n",
       " (Gizmodo, 0.14929114),\n",
       " (Ukip, 0.14707668),\n",
       " (Holmes, 0.14671838),\n",
       " (Brown, 0.14635204),\n",
       " (Swagway, 0.14633487),\n",
       " (Andy Nunn, 0.14584056),\n",
       " (Brexiters, 0.14581239),\n",
       " (Brown, 0.14487797),\n",
       " (Thom Yorke, 0.14487423),\n",
       " (Muriel Schofield, 0.1447751),\n",
       " (Alli, 0.14440942),\n",
       " (Edge, 0.14408046),\n",
       " (John Higgins, 0.14353634),\n",
       " (Bethan Wolfenden, 0.14247653),\n",
       " (Caldicott, 0.14178956),\n",
       " (Tidal, 0.1413688),\n",
       " (Tesla, 0.14030042),\n",
       " (Sue Torka, 0.14000946),\n",
       " (Cannes, 0.13976444),\n",
       " (Derren Brown, 0.13864067),\n",
       " (Counterstrike, 0.13844761),\n",
       " (Stephen Foreshew-Cain, 0.13797012),\n",
       " (Stefan Beller, 0.13782676),\n",
       " (Kira Nelson, 0.13681503),\n",
       " (Brown, 0.1364114),\n",
       " (Walgreens, 0.13605085),\n",
       " (Toffler, 0.13510378),\n",
       " (McCoy, 0.13509718),\n",
       " (Clinton, 0.13451381),\n",
       " (Isis, 0.13444702),\n",
       " (Smales, 0.13423966),\n",
       " (Bear, 0.13400477),\n",
       " (Tidal, 0.13372496),\n",
       " (Bruce, 0.13276124),\n",
       " (Tidal, 0.13230243),\n",
       " (Stranger, 0.13179357),\n",
       " (Thatcher, 0.13174267),\n",
       " (Toffler, 0.13143706),\n",
       " (Tishin Donkersley, 0.13086072),\n",
       " (Smales, 0.130603),\n",
       " (Spotify, 0.13057192),\n",
       " (Buzzbike, 0.13026224),\n",
       " (Nick Bilton, 0.12997313),\n",
       " (Aimoto, 0.12976377),\n",
       " (Tim, 0.12892538),\n",
       " (Jay Z, 0.12842761),\n",
       " (Victoria Barracks, 0.12805192),\n",
       " (Marc Benioff, 0.1279304),\n",
       " (Louis Theroux, 0.12734367),\n",
       " (Zausen, 0.12718968),\n",
       " (Jake Dyson, 0.12687197),\n",
       " (Steve Jobs, 0.1267443),\n",
       " (Brown, 0.12669972),\n",
       " (Wiley, 0.12651046),\n",
       " (Brown, 0.1264123),\n",
       " (Harbisson, 0.12529983),\n",
       " (Tidal, 0.12523215),\n",
       " (Alvin, 0.12486246),\n",
       " (Jay Z, 0.12471751),\n",
       " (Holmes, 0.12426841),\n",
       " (Autopilot, 0.124076344),\n",
       " (Laurence Peter, 0.12407119),\n",
       " (KitKat, 0.123414256),\n",
       " (Tesla, 0.12262823),\n",
       " (Tesla, 0.12239852),\n",
       " (Spotify, 0.1221667),\n",
       " (Brown, 0.12183704),\n",
       " (Borlongan, 0.12176141),\n",
       " (uMotif, 0.12108979),\n",
       " (Boris, 0.12066559),\n",
       " (Berlin Partner, 0.120610885),\n",
       " (Alvin, 0.12060523),\n",
       " (Sridhar Lakshmanan, 0.12035886),\n",
       " (Eric Mar, 0.12015633),\n",
       " (Mustafa, 0.11994799),\n",
       " (Fraunhofer, 0.11987933),\n",
       " (Lovell, 0.11981497),\n",
       " (Sung-Jin Park, 0.11952264),\n",
       " (Follow Guardian, 0.11945668),\n",
       " (Moon Ribas, 0.11935065),\n",
       " (Bush, 0.11837625),\n",
       " (Autopilot, 0.11794766),\n",
       " (Baressi, 0.11769013),\n",
       " (Tom Morgan, 0.11628754),\n",
       " (onTwitter, 0.11607969),\n",
       " (Harry Potter, 0.115567334),\n",
       " (Gibney, 0.11474894),\n",
       " (Lovell, 0.114672706),\n",
       " (Don MacKenzie, 0.114500836),\n",
       " (Simon Chapman, 0.113933064),\n",
       " (Karl Brauer, 0.11377082),\n",
       " (Tommy Vercetti, 0.11371716),\n",
       " (Zooniverse, 0.11314238),\n",
       " (Harry Potter, 0.11293509),\n",
       " (Holmes, 0.112323314),\n",
       " (Paul Clarke, 0.110999554),\n",
       " (John Major, 0.110798255),\n",
       " (Machiavelli, 0.11040403),\n",
       " (Wi-Fi, 0.110293105),\n",
       " (Harry, 0.11006215),\n",
       " (Gundam, 0.109303616),\n",
       " (Alvin Eugene Toffler, 0.108800314),\n",
       " (Dave Gilboa, 0.108354025),\n",
       " (Brown, 0.10760995),\n",
       " (Pushed, 0.10759363),\n",
       " (Corbyn, 0.10713306),\n",
       " (Cuisinart, 0.107062235),\n",
       " (NHS England, 0.10636933),\n",
       " (Buzzbike, 0.106288485),\n",
       " (Kanye West, 0.10564315),\n",
       " (Swagway X1, 0.10550695),\n",
       " (Buzzbike, 0.105348356),\n",
       " (Tom Hares, 0.10532404),\n",
       " (McElroy, 0.10520741),\n",
       " (Calorie Counter, 0.103838496),\n",
       " (Patrick Seidel, 0.103275135),\n",
       " (Tidal, 0.10321734),\n",
       " (Dyson, 0.103095695),\n",
       " (Tesla, 0.102942),\n",
       " (Madonna, 0.10269465),\n",
       " (Audrey Hepburn, 0.10253589),\n",
       " (Larry Page, 0.102367826),\n",
       " (Rose, 0.102162756),\n",
       " (Davison, 0.10158617),\n",
       " (Robin Ince, 0.10105643),\n",
       " (Vivid, 0.10090074),\n",
       " (PokéStop, 0.10047323),\n",
       " (Bradley Wiggins, 0.100360595),\n",
       " (Elon Musk, 0.10005843),\n",
       " (Moto Insta-Share, 0.0996754),\n",
       " (Nitro Zeus, 0.099546604),\n",
       " (Tesla, 0.09933554),\n",
       " (Wi-Fi, 0.098841764),\n",
       " (Philip Green, 0.09727976),\n",
       " (Corbyn, 0.09706598),\n",
       " (Snap, 0.09672078),\n",
       " (Netflix, 0.0966552),\n",
       " (Corbyn, 0.09640992),\n",
       " (Ashley Madison, 0.09640118),\n",
       " (Bitcoin, 0.095212944),\n",
       " (Adam McKay, 0.09496911),\n",
       " (Respawn, 0.09496693),\n",
       " (Holmes, 0.09459256),\n",
       " (Trak, 0.09391391),\n",
       " (Rohos, 0.09345121),\n",
       " (Mikhail Gorbachev, 0.093385816),\n",
       " (Max Planck, 0.092490174),\n",
       " (Respawn, 0.09227121),\n",
       " (Tidal, 0.091282),\n",
       " (Bluedot, 0.0908807),\n",
       " (Twitter, 0.09085653),\n",
       " (Stephen Pinker, 0.09034296),\n",
       " (Brown, 0.08983553),\n",
       " (Tesla, 0.089664765),\n",
       " (Harry Potter, 0.08950708),\n",
       " (Wallace, 0.089045815),\n",
       " (Xi Jinping, 0.089025825),\n",
       " (Zheng Xiaonian, 0.088853136),\n",
       " (Kurt, 0.08877031),\n",
       " (Brown, 0.08851183),\n",
       " (Apples, 0.088395275),\n",
       " (James Dyson, 0.08829312),\n",
       " (Watson, 0.08823826),\n",
       " (Londonist, 0.087763645),\n",
       " (Brown, 0.08759986),\n",
       " (Shashi Verma, 0.08718685),\n",
       " (Angry Birds Transformers, 0.086899154),\n",
       " (Cranfield’s, 0.08661529),\n",
       " (Peff, 0.08621859),\n",
       " (Tesla, 0.08527055),\n",
       " (Lancashire, 0.08440462),\n",
       " (Clarke, 0.08400939),\n",
       " (Feng Kung, 0.08383693),\n",
       " (Lidar, 0.083019465),\n",
       " (Ray, 0.08237732),\n",
       " (Autopilot, 0.08199173),\n",
       " (Macauley Culkin, 0.08187258),\n",
       " (Angry Birds, 0.081855156),\n",
       " (Shepherd, 0.08148266),\n",
       " (Orson Welles, 0.08135736),\n",
       " (Danny Leigh’s, 0.081249245),\n",
       " (Stranger, 0.08112913),\n",
       " (Khan, 0.079940185),\n",
       " (Sarah Templey, 0.079700395),\n",
       " (David Cameron, 0.07942777),\n",
       " (Ash Ketchum, 0.07937138),\n",
       " (Toffler, 0.07891761),\n",
       " (Kyle Evans, 0.07868955),\n",
       " (Talulah Riley, 0.078235045),\n",
       " (Toffler, 0.07804966),\n",
       " (Peter Bradshaw, 0.07742387),\n",
       " (Buzzbike, 0.076878384),\n",
       " (Vitalik Buterin, 0.076757155),\n",
       " (Megadeth, 0.07661947),\n",
       " (Gmail, 0.07626822),\n",
       " (Toffler, 0.07602967),\n",
       " (Nunn, 0.075624846),\n",
       " (Ashley Madison, 0.0755576),\n",
       " (Brown, 0.075507484),\n",
       " (Autopilot, 0.07526017),\n",
       " (Brown, 0.07524547),\n",
       " (Blizzard, 0.0751666),\n",
       " (Edmonton, 0.07443315),\n",
       " (Nougat, 0.07430231),\n",
       " (Respawn, 0.07396523),\n",
       " (Powerboard, 0.07381518),\n",
       " (Ribas, 0.07353522),\n",
       " (Catherine Deneuve, 0.073463544),\n",
       " (Stuxnet, 0.07338724),\n",
       " (Gibney, 0.072532795),\n",
       " (Matthew Garrett, 0.07178502),\n",
       " (Rajkumar, 0.07172189),\n",
       " (Naidenov, 0.071552806),\n",
       " (Vivaldi, 0.0707081),\n",
       " (Sam Singer, 0.07061951),\n",
       " (McCoy, 0.07046824),\n",
       " (BBC North, 0.07021303),\n",
       " (Moto Z, 0.07017085),\n",
       " (Sarah Vine’s, 0.069717586),\n",
       " (Michael, 0.06941294),\n",
       " (Kevin Kit Parker, 0.06927884),\n",
       " (Aimoto, 0.06891578),\n",
       " (Jack Russell, 0.06878443),\n",
       " (Ben Robinson, 0.06853701),\n",
       " (Phonebloks, 0.06852019),\n",
       " (Missoffe, 0.068316445),\n",
       " (Warby Parker’s, 0.06825168),\n",
       " (Amelias, 0.06808638),\n",
       " (Toffler, 0.06803244),\n",
       " (Diamond, 0.06778281),\n",
       " (Korine, 0.067280754),\n",
       " (Milano, 0.06709017),\n",
       " (Rzevski, 0.06704774),\n",
       " (Walker, 0.0664379),\n",
       " (Bob Greifeld, 0.06639075),\n",
       " (Wallace, 0.06606282),\n",
       " (McCoy, 0.06555992),\n",
       " (Nunn, 0.06498687),\n",
       " (Lidar, 0.06458007),\n",
       " (Ashley Madison, 0.06423348),\n",
       " (Dan Orloff, 0.06410638),\n",
       " (Buzzbike, 0.06407348),\n",
       " (Gibney, 0.06380779),\n",
       " (Toffler, 0.0637194),\n",
       " (Wi-Fi, 0.06335336),\n",
       " (eSports, 0.063302554),\n",
       " (Banks, 0.06283453),\n",
       " (Bluedot, 0.06254677),\n",
       " (Tesla, 0.06249216),\n",
       " (Gyms, 0.062414385),\n",
       " (Mark Jenkins, 0.061409906),\n",
       " (AuYou Wi-Fi, 0.06120923),\n",
       " (Richard Wallace, 0.06104587),\n",
       " (Check Point, 0.06100703),\n",
       " (Top Gear, 0.060770262),\n",
       " (Hearthstone, 0.06038156),\n",
       " (Vivaldi, 0.059778012),\n",
       " (Calvin Harris, 0.059352428),\n",
       " (Deborah, 0.059226263),\n",
       " (Amsterdam, 0.05872846),\n",
       " (Stranger, 0.0587045),\n",
       " (Cooper, 0.058590382),\n",
       " (Barcode, 0.057240166),\n",
       " (Jones, 0.05678672),\n",
       " (Baidus, 0.05598824),\n",
       " (Musk, 0.05581102),\n",
       " (Eton, 0.054844208),\n",
       " (Tropico, 0.054693986),\n",
       " (Missoffe, 0.054608237),\n",
       " (PokéStop, 0.054477222),\n",
       " (Bruce Hellman, 0.053830102),\n",
       " (Davison, 0.053426996),\n",
       " (Guardian Cities, 0.05313905),\n",
       " (Tesla, 0.051749565),\n",
       " (Anna Lauren Hoffmann, 0.0517107),\n",
       " (George Stephenson’s, 0.05124366),\n",
       " (Andrew Brandt, 0.05081143),\n",
       " (Geralt, 0.050559297),\n",
       " (Tim Yane, 0.050302178),\n",
       " (Avenue Q.\n",
       "  , 0.050213166),\n",
       " (Nougat, 0.050209012),\n",
       " (Narcissus, 0.05015447),\n",
       " (Kurt Coleman, 0.049852237),\n",
       " (Charlie Chaplin, 0.04962363),\n",
       " (Taavet Hinrikus, 0.04926094),\n",
       " (Miguel, 0.049185827),\n",
       " (Trent Reznor, 0.04854321),\n",
       " (Harbisson, 0.0485177),\n",
       " (Diagnosis, 0.048081685),\n",
       " (” Zheng, 0.047910057),\n",
       " (Ed Miliband, 0.0475195),\n",
       " (John Foley, 0.047000695),\n",
       " (Warby Parker, 0.04697338),\n",
       " (Kevin Bannon, 0.04639386),\n",
       " (Donald Trump, 0.04631877),\n",
       " (Baczuk, 0.045729738),\n",
       " (Jonathan Petit, 0.045729365),\n",
       " (Check Point’s, 0.045239),\n",
       " (Walgreens, 0.04472261),\n",
       " (Autopilot, 0.044355635),\n",
       " (Elon Musk, 0.04393827),\n",
       " (Rihanna, 0.043836854),\n",
       " (Elon Musk, 0.043678336),\n",
       " (Tidal, 0.04318701),\n",
       " (Reid, 0.043173175),\n",
       " (Kakade, 0.042794287),\n",
       " (LG Cam, 0.042609457),\n",
       " (Described, 0.042262394),\n",
       " (Stefan, 0.04224352),\n",
       " (WiFi, 0.04217629),\n",
       " (Crabby, 0.0421645),\n",
       " (Hoffmann, 0.042073254),\n",
       " (Wikipedia, 0.041751783),\n",
       " (Android 6 Marshmallow, 0.04136611),\n",
       " (Simon Pegg, 0.04123398),\n",
       " (James Dyson, 0.040735405),\n",
       " (Thatcher, 0.040388606),\n",
       " (Harbisson, 0.03963575),\n",
       " (Joshua Brown, 0.039614413),\n",
       " (Spotify, 0.038836025),\n",
       " (iOS, 0.038725443),\n",
       " (Lidar, 0.038646687),\n",
       " (Hares, 0.03860237),\n",
       " (Trak, 0.03835698),\n",
       " (Leonardo DiCaprio, 0.038282398),\n",
       " (IoT, 0.036798574),\n",
       " (Ashley Madison, 0.036577594),\n",
       " (Michael Gove, 0.03630253),\n",
       " (Autopilot, 0.036237556),\n",
       " (Kieran, 0.036036648),\n",
       " (Chris Martin, 0.035965197),\n",
       " (Jones, 0.034725964),\n",
       " (Vandana Shiva, 0.034407195),\n",
       " (Brown, 0.03418899),\n",
       " (Jay Z, 0.034110636),\n",
       " (Articles, 0.033620086),\n",
       " (the Cure Parkinson’s, 0.033583026),\n",
       " (Netanyahu, 0.033371855),\n",
       " (Tesla, 0.0331185),\n",
       " (Tesla Autopilot, 0.032463174),\n",
       " (Lidar, 0.032307543),\n",
       " (Mario, 0.032227628),\n",
       " (Yingmob, 0.031923313),\n",
       " (Josh, 0.031800568),\n",
       " (Devra Davis, 0.031563163),\n",
       " (Woody Allen, 0.031348556),\n",
       " (Ray, 0.030181987),\n",
       " (Lyndon Rive, 0.029680457),\n",
       " (The Darkest Spray Tan, 0.029313926),\n",
       " (Corbyn, 0.029035287),\n",
       " (Jordan Baczuk, 0.027817031),\n",
       " (Boris Johnson, 0.027578047),\n",
       " (William Huskisson, 0.027550723),\n",
       " (Kurt Coleman, 0.027145725),\n",
       " (Johnson, 0.02551785),\n",
       " (Dan Wiley, 0.025511399),\n",
       " (Zoё Kelly, 0.025165234),\n",
       " (Lidar, 0.023784155),\n",
       " (” Naidenov, 0.023692895),\n",
       " (Greg Kroah-Hartman, 0.023610674),\n",
       " (Farid Yaghoubtil, 0.023242882),\n",
       " (Warby Parker, 0.022625849),\n",
       " (Garrett, 0.022546388),\n",
       " (Steve Hilton, 0.022540951),\n",
       " (Wiley, 0.022429753),\n",
       " (Saw, 0.022073135),\n",
       " (Chewbacca, 0.021915756),\n",
       " (Militia, 0.021734338),\n",
       " (Brown, 0.021721164),\n",
       " (Peff, 0.021674987),\n",
       " (Dame Fiona Caldicott, 0.021594584),\n",
       " (Joshua Brown, 0.021590933),\n",
       " (Drake, 0.02156829),\n",
       " (Moorfields, 0.02107541),\n",
       " (Android, 0.020820143),\n",
       " (Peng Tee Khaw, 0.020656852),\n",
       " (Picking Challenge, 0.02063156),\n",
       " (Talulah Riley, 0.02029144),\n",
       " (Swagway LLC, 0.02008758),\n",
       " (Rihanna, 0.020064026),\n",
       " (Harbisson, 0.019663533),\n",
       " (Sandro, 0.018767921),\n",
       " (Edmonton, 0.018641155),\n",
       " (Cameron, 0.018570958),\n",
       " (Wallace, 0.018428314),\n",
       " (Royal Free, 0.017946677),\n",
       " (Michael Perera, 0.017881788),\n",
       " (Harbisson, 0.017853219),\n",
       " (Autopilot, 0.017464561),\n",
       " (Ultimate Guitar, 0.017239418),\n",
       " (Drake, 0.017064981),\n",
       " (Singh Kwatra, 0.016970605),\n",
       " (Mikhail Trutnev, 0.016848482),\n",
       " (YouTube, 0.016642576),\n",
       " (Shield, 0.016163956),\n",
       " (Caldicott, 0.015685162),\n",
       " (Albert Scaglione, 0.015198346),\n",
       " (Freddie Starr, 0.014486988),\n",
       " (Lidar, 0.014484226),\n",
       " (Nunn, 0.01445353),\n",
       " (David Carr, 0.014115006),\n",
       " (Delft, 0.014000201),\n",
       " (Spotify, 0.013875237),\n",
       " (Warby Parker, 0.013751145),\n",
       " (Anthony Johnson, 0.013583889),\n",
       " (Tidal, 0.013456505),\n",
       " (Carr, 0.013108022),\n",
       " (Jake Dyson, 0.012461157),\n",
       " (N. Most previous versions, 0.012318602),\n",
       " (Dom Smales, 0.0122198565),\n",
       " (Rodeo Stampede, 0.0121887345),\n",
       " (Android 4.4, 0.011891397),\n",
       " (Tidal, 0.010529005),\n",
       " (Spotify, 0.010357729),\n",
       " (Bill Gates, 0.010012382),\n",
       " (Tidal, 0.009720507),\n",
       " (the Ghost Train, 0.00964671),\n",
       " (Khan, 0.009606687),\n",
       " (Markit, 0.009593154),\n",
       " (Brown, 0.009444764),\n",
       " (Mar, 0.00871675),\n",
       " (Buzzbike, 0.008088198),\n",
       " (Panahi, 0.00743939),\n",
       " (Nelson, 0.007419099),\n",
       " (Woo, 0.0071685617),\n",
       " (Karen, 0.0067700483),\n",
       " (Michael Gove, 0.0062881676),\n",
       " (William Davies, 0.0061400025),\n",
       " (mm, 0.005685662),\n",
       " (Debra Charles, 0.0053037475),\n",
       " (PokéStop, 0.004579366),\n",
       " (David, 0.004111438),\n",
       " (Zipcar car-sharing, 0.0029706112),\n",
       " (Buzzbike, 0.0029060512),\n",
       " (Derek Zoolander, 0.0027992483),\n",
       " (Warby Parker, 0.002731617),\n",
       " (Merlin, 0.0024059506),\n",
       " (NHS England, 0.0019386756),\n",
       " (Marshall, 0.0015038783),\n",
       " (Bluedot, 0.0014987028),\n",
       " (Rupert Murdoch, 0.0014505252),\n",
       " (Mike Butcher, 0.0012811723),\n",
       " (Doom, 0.0004453288),\n",
       " (McCoy, -0.00031577895),\n",
       " (Lidar, -0.0004234494),\n",
       " (The I Sea, -0.00042407884),\n",
       " (Mark Zuckerberg, -0.0006361751),\n",
       " (Richard Wallace, -0.0009592941),\n",
       " (Patrick Reynolds, -0.0012757733),\n",
       " (Holmes, -0.0016950836),\n",
       " (Jean-Michel Jarre, -0.0017547245),\n",
       " (Indira Gandhi, -0.0030486055),\n",
       " (Nasty Gal, -0.0031939773),\n",
       " (Autopilot, -0.003862125),\n",
       " (Nithiyanandam, -0.0039670723),\n",
       " (Pablo Picasso, -0.0042417864),\n",
       " (Spotify, -0.004803024),\n",
       " (Gay Shame, -0.0053578597),\n",
       " (Autopilot, -0.0055592014),\n",
       " (Warby Parker’s, -0.005694448),\n",
       " (Article, -0.006195843),\n",
       " (John McElroy, -0.006740571),\n",
       " (Netflix, -0.0068399603),\n",
       " (Joshua Brown’s, -0.0069001126),\n",
       " (Elon Musk, -0.0069188466),\n",
       " (Marcus Butler, -0.007301319),\n",
       " (Nunn, -0.00776314),\n",
       " (Coherence Tomography, -0.0079404935),\n",
       " (Take Chris Martin, -0.007955658),\n",
       " (Brown, -0.007974308),\n",
       " (Barrett, -0.008410172),\n",
       " (Gordon Brown, -0.008507686),\n",
       " (Musk’s, -0.0088833105),\n",
       " (Windows, -0.009130752),\n",
       " (Karen Levy, -0.009149544),\n",
       " (Blackberry, -0.009479133),\n",
       " (Frank Baressi, -0.01085687),\n",
       " (Moon, -0.011668024),\n",
       " (Jobs, -0.012170698),\n",
       " (Fi Plus, -0.012221197),\n",
       " (Linus Torvalds, -0.012447456),\n",
       " (Brandt, -0.012784888),\n",
       " (Wiley, -0.0136138555),\n",
       " (Buzzbike, -0.0138224345),\n",
       " (Diego Gigliani, -0.014022974),\n",
       " (Privacy Shield, -0.014854002),\n",
       " (Wallace, -0.015593167),\n",
       " (Ragunathan Rajkumar, -0.015935825),\n",
       " (Christopher Nolan, -0.016347203),\n",
       " (Denzel Smith, -0.016350588),\n",
       " (Alan Garner, -0.016486943),\n",
       " (Zoe “Zoella” Sugg, -0.016808767),\n",
       " (Tesla, -0.016939227),\n",
       " (Autopilot, -0.016942829),\n",
       " (Madonna, -0.017143533),\n",
       " (Sherlock Holmes, -0.017534805),\n",
       " (Tidal, -0.017586136),\n",
       " (Autopilot, -0.01789143),\n",
       " (Hyperloop, -0.018277662),\n",
       " (Toffler, -0.018708432),\n",
       " (Jimmy Iovine, -0.020018822),\n",
       " (Gawker, -0.020277284),\n",
       " (Isis, -0.020388523),\n",
       " (Wi-Fried, -0.020513503),\n",
       " (Hugo Chávez, -0.020696107),\n",
       " (Tesla, -0.02075311),\n",
       " (Autopilot, -0.020812288),\n",
       " (Balderton Capital, -0.021087606),\n",
       " (Lidar, -0.021144023),\n",
       " (Ian Morris, -0.021558017),\n",
       " (PokéCoins, -0.02176469),\n",
       " (Trak, -0.022049969),\n",
       " (Tidal, -0.02248623),\n",
       " (Check Point, -0.022507105),\n",
       " (Gibney, -0.022693753),\n",
       " (Elon Musk, -0.02283198),\n",
       " (Samuel Ely, -0.02296951),\n",
       " (Michael Bautista, -0.02331907),\n",
       " (Stanley Kubrick, -0.023348518),\n",
       " (Jelly Bean, -0.023395268),\n",
       " (Tesla, -0.024848916),\n",
       " (Sam, -0.025019005),\n",
       " (AI, -0.025723334),\n",
       " (Max Conze, -0.025909675),\n",
       " (Cassie, -0.026666302),\n",
       " (Jay Z, -0.0267579),\n",
       " (PS4, -0.026791811),\n",
       " (\n",
       "  \n",
       "  submitGit, -0.026889877),\n",
       " (Jeremy Corbyn, -0.02715182),\n",
       " (Hares, -0.027475227),\n",
       " (Bitcoin, -0.0275024),\n",
       " (Kanye West, -0.028988063),\n",
       " (Dave, -0.02925215),\n",
       " (Ray, -0.029992582),\n",
       " (Respawn, -0.030246008),\n",
       " (Respawn, -0.030477213),\n",
       " (Greenpeace, -0.03142733),\n",
       " (\n",
       "  \n",
       "  Levy, -0.032063883),\n",
       " (Selfie, -0.03273238),\n",
       " (Blair, -0.0330893),\n",
       " (Wolverhampton, -0.033255365),\n",
       " (Billed, -0.033297595),\n",
       " (Jade Davis, -0.03360071),\n",
       " (Ramesh Kakade, -0.033842027),\n",
       " (Naidenov, -0.034116462),\n",
       " (Burton, -0.03462058),\n",
       " (Bounty Hunt, -0.034702383),\n",
       " (Wi-Fi, -0.03508856),\n",
       " (Meerkat, -0.03554143),\n",
       " (\n",
       "  \n",
       "  Smith, -0.035935096),\n",
       " (Andy, -0.03637739),\n",
       " (Brian Eno, -0.036722064),\n",
       " (Tidal, -0.03701331),\n",
       " (Twitter, -0.03707066),\n",
       " (Ed Lee, -0.038299523),\n",
       " (Elena Gaytan, -0.038795527),\n",
       " (David Willetts, -0.039407324),\n",
       " (Jelly Bean, -0.041274678),\n",
       " (Stuxnet, -0.041825403),\n",
       " (Liz McInnes, -0.042043034),\n",
       " (James Millership, -0.042622298),\n",
       " (Amec Foster Wheeler, -0.042713664),\n",
       " (Brown, -0.043542862),\n",
       " (Respawn, -0.04418041),\n",
       " (Jacques de la Bouillerie, -0.04433284),\n",
       " (Ferdinand Dudenhöffer, -0.044521417),\n",
       " (Robinson, -0.045215618),\n",
       " (Lesley Xavier, -0.045481868),\n",
       " (Amazon Fresh, -0.04548777),\n",
       " (Mobiles, -0.04576403),\n",
       " (Lidar, -0.04584719),\n",
       " (Paddy Dickinson, -0.04585429),\n",
       " (Lidar, -0.046734393),\n",
       " (Bloomberg, -0.047617774),\n",
       " (Kanye West, -0.048288792),\n",
       " (Lemon, -0.04844299),\n",
       " (Tesla, -0.049619377),\n",
       " (Mark Zuckerberg, -0.04967669),\n",
       " (Ashley Madison, -0.049857758),\n",
       " (Musk, -0.05043149),\n",
       " (Waiting, -0.050642405),\n",
       " (Latino, -0.05145589),\n",
       " (Twitter, -0.0517514),\n",
       " (Lovell, -0.051915832),\n",
       " (Rdio, -0.052917663),\n",
       " (Johnson, -0.05309923),\n",
       " (Policy Connect, -0.053302612),\n",
       " (Ralph Pini, -0.05354809),\n",
       " (Kings Place, -0.05514958),\n",
       " (Alicia Keys, -0.05579641),\n",
       " (Humanitas, -0.055949368),\n",
       " (Ours, -0.056010235),\n",
       " (Jenkins, -0.056813464),\n",
       " (Jeff Bezos, -0.057464324),\n",
       " (Wi-Fi, -0.057591088),\n",
       " (Cocoa, -0.059226483),\n",
       " (Alisher Usmanov, -0.06030831),\n",
       " (Ed Miliband’s, -0.06090954),\n",
       " (Trak, -0.061142024),\n",
       " (Brogrammers, -0.06366879),\n",
       " (David Lynch, -0.06377116),\n",
       " (Theranos, -0.06440964),\n",
       " (Gibney, -0.06476345),\n",
       " (Jennifer Lawrence, -0.0647926),\n",
       " (Kairosoft, -0.06559195),\n",
       " (Trak, -0.06576121),\n",
       " (Pearse Keane, -0.066005126),\n",
       " (George Rzevski, -0.06652521),\n",
       " (Brown, -0.06791727),\n",
       " (Peter Alliss, -0.06792631),\n",
       " (Alex Reid, -0.067934066),\n",
       " (Edmonton, -0.06795446),\n",
       " (Davison, -0.0679729),\n",
       " (McCoy, -0.068184786),\n",
       " (Ashley Madison, -0.06987858),\n",
       " (Gove, -0.06995836),\n",
       " (Hares, -0.07197907),\n",
       " (Todd, -0.07247457),\n",
       " (Deborah Lupton, -0.07278681),\n",
       " (Dickinson, -0.07518285),\n",
       " (Respawn, -0.076045044),\n",
       " (Tidal, -0.07849611),\n",
       " (Titanfall, -0.080582164),\n",
       " (Tom Watson, -0.08200776),\n",
       " (Android, -0.08257273),\n",
       " (Musk, -0.08326527),\n",
       " (Murdoch, -0.08426962),\n",
       " (Ashley Madison, -0.085346796),\n",
       " (Twitter, -0.08560645),\n",
       " (Vera Jourova, -0.086816296),\n",
       " (Carren Jao, -0.08702167),\n",
       " (Raul Diaz, -0.08733453),\n",
       " (Dave Mustaine, -0.08791269),\n",
       " (Helen Keen, -0.0881646),\n",
       " (Seidel, -0.08818601),\n",
       " (Zuckerberg, -0.08965261),\n",
       " (Carbine, -0.08980897),\n",
       " (Taru Kapoor, -0.090177536),\n",
       " (Uncle Pete, -0.09105382),\n",
       " (vehicles’, -0.09413682),\n",
       " (Brian Colello, -0.09430951),\n",
       " (Self Portrait, -0.09447085),\n",
       " (Mark Hughes, -0.09476917),\n",
       " (Check Point, -0.09495681),\n",
       " (Brian Cox, -0.09564071),\n",
       " (Sandro, -0.098111376),\n",
       " (Garrett, -0.09829361),\n",
       " (Merlin, -0.098735854),\n",
       " (Pranit Bauva, -0.104149275),\n",
       " (Jonathan Caouette, -0.105196625),\n",
       " (Twitter, -0.10616057),\n",
       " (David Newby, -0.10723272),\n",
       " (Forelli, -0.1080212),\n",
       " (Lidar, -0.10859532),\n",
       " (Andrew Fentem, -0.10976928),\n",
       " (Cornelia Yzer, -0.11016309),\n",
       " (Tidal, -0.11043357),\n",
       " (Peasant Children, -0.111250035),\n",
       " (Travis Todd, -0.111894146),\n",
       " (Michelle Du-Prat, -0.11307976),\n",
       " (Slavoj Zizek, -0.116698876),\n",
       " (Panahi, -0.12170073),\n",
       " (Bluedot, -0.12262855),\n",
       " (Wiley, -0.12514026),\n",
       " (Harry Briggs, -0.12515673),\n",
       " (Stuxnet, -0.1263243),\n",
       " (Miller, -0.12729967),\n",
       " (Jay Z, -0.12736776),\n",
       " (Damian Kimmelman, -0.13315944),\n",
       " (Tim Peake, -0.13722974),\n",
       " (Hovertrax, -0.1391795),\n",
       " (Nebula, -0.14096387),\n",
       " (Wendy Berliner, -0.14309822),\n",
       " (Tidal, -0.14434704),\n",
       " (Jay Z, -0.14542662),\n",
       " (Alton Towers, -0.14788494),\n",
       " (Ari Bloom, -0.15031445),\n",
       " (Snap, -0.15479144),\n",
       " (Yogita Suryavanshi, -0.15512192),\n",
       " (Pidgey, -0.1582908),\n",
       " (\n",
       "  \n",
       "  Smartphone, -0.15966251),\n",
       " (Borgias, -0.1625935),\n",
       " (Jack, -0.17006944),\n",
       " (William Oliver Healey, -0.17517976),\n",
       " (Harbisson, -0.17961945),\n",
       " (Toca Boca, -0.19228205),\n",
       " (Kyra Kazantzis, -0.2022153),\n",
       " (Tidal, -0.20813188),\n",
       " (Jonathan Bernstein, -0.21280837),\n",
       " (Ghost Train, -0.21948896),\n",
       " (Neil Blumenthal, -0.22184233)]"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(similarities,key= lambda x: x[1], reverse=True)\n",
    "\n",
    "#Słabo działa, spróbujemy z większymi wektorami"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoring=[]\n",
    "for doc in df['doc']:\n",
    "    if('Jay Z' in doc.text):\n",
    "        scoring.append(doc.sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.12821747362613678, 0.05493183061480522]"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoring"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting extension to sentence level polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.tokens import Span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def polarity(span):\n",
    "    return analyzer.analyze(span.text).polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "Span.set_extension('polarity', getter=polarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04999999999999999\n",
      "0.0\n",
      "0.0\n",
      "-0.4\n",
      "0.0\n",
      "-0.3499999999999999\n",
      "-0.35\n",
      "-0.2375\n",
      "-0.2333333333333333\n",
      "-0.11851851851851852\n",
      "0.25\n",
      "0.0\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "for sent in doc.sents:\n",
    "    print(sent._.polarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textacy.lexicon_methods import emotional_valence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "textacy.lexicon_methods.download_depechemood()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(float,\n",
       "            {'AFRAID': 0.13200744218749993,\n",
       "             'AMUSED': 0.14212075588202255,\n",
       "             'ANGRY': 0.1158086805909092,\n",
       "             'ANNOYED': 0.11762475423295463,\n",
       "             'DONT_CARE': 0.11652026924999999,\n",
       "             'HAPPY': 0.12144892282162159,\n",
       "             'INSPIRED': 0.1563725598432433,\n",
       "             'SAD': 0.12797938725543484})"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emotional_valence(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}