Named Entity Recognition .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tag import pos_tag\n",
    "from nltk.chunk import ne_chunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'Zidane agreed to take charge of Real until the summer of 2022. His love for the club he had graced as both a player and a coach, he said, was too great for him to resist. “I could not say no,” he said. “I am here because the president called me.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ne_tree = ne_chunk(pos_tag(word_tokenize(sentence)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ne_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex = 'Zidane agreed to take charge of Real until the summer of 2022. His love for the club he had graced as both a player and a coach, he said, was too great for him to resist. “I could not say no,” he said. “I am here because the president called me.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sent):\n",
    "    sent = nltk.word_tokenize(sent)\n",
    "    sent = nltk.pos_tag(sent)\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = preprocess(ex)\n",
    "sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = 'NP: {<DT>?<JJ>*<NN>}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp = nltk.RegexpParser(pattern)\n",
    "cs = cp.parse(sent)\n",
    "print(cs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NPChunker = nltk.RegexpParser(pattern) \n",
    "result = NPChunker.parse(sent)\n",
    "result.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.chunk import conlltags2tree, tree2conlltags\n",
    "from pprint import pprint\n",
    "\n",
    "iob_tagged = tree2conlltags(cs)\n",
    "pprint(iob_tagged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "from collections import Counter\n",
    "import en_core_web_sm\n",
    "nlp = en_core_web_sm.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp('Zidane agreed to take charge of Real until the summer of 2022. His love for the club he had graced as both a player and a coach, he said, was too great for him to resist. “I could not say no,” he said. “I am here because the president called me.')\n",
    "pprint([(X.text, X.label_) for X in doc.ents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint([(X, X.ent_iob_, X.ent_type_) for X in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def url_to_string(url):\n",
    "    res = requests.get(url)\n",
    "    html = res.text\n",
    "    soup = BeautifulSoup(html, 'html5lib')\n",
    "    for script in soup([\"script\", \"style\", 'aside']):\n",
    "        script.extract()\n",
    "    return \" \".join(re.split(r'[\\n\\t]+', soup.get_text()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ny_bb = url_to_string('https://www.nytimes.com/2019/03/11/sports/real-madrid-zinedine-zidane.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=first-column-region&region=top-news&WT.nav=top-news')\n",
    "article = nlp(ny_bb)\n",
    "len(article.ents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [x.label_ for x in article.ents]\n",
    "Counter(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "items = [x.text for x in article.ents]\n",
    "Counter(items).most_common(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = [x for x in article.sents]\n",
    "print(sentences[20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "displacy.render(nlp(str(sentences[20])), jupyter=True, style='ent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "displacy.render(nlp(str(sentences[20])), style='dep', jupyter = True, options = {'distance': 120})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[(x.orth_,x.pos_, x.lemma_) for x in [y \n",
    "                                      for y\n",
    "                                      in nlp(str(sentences[20])) \n",
    "                                      if not y.is_stop and y.pos_ != 'PUNCT']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict([(str(x), x.label_) for x in nlp(str(sentences[20])).ents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print([(x, x.ent_iob_, x.ent_type_) for x in sentences[20]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "displacy.render(article, jupyter=True, style='ent')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}