create-rules.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "90d52226",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting en-core-web-sm==3.3.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 12.8 MB 1.4 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from en-core-web-sm==3.3.0) (3.3.0)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.7.7)\n",
      "Requirement already satisfied: setuptools in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (58.0.4)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.62.3)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.26.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.15)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
      "Requirement already satisfied: jinja2 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.11.3)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.20.3)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.4)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.10.0.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.26.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.2)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.3)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /Users/andy.gout/opt/anaconda3/lib/python3.9/site-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.1.1)\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b580446b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import spacy\n",
    "from spacy.pipeline import EntityRuler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "528baf2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(file):\n",
    "    with open (file, \"r\", encoding=\"utf-8\") as f:\n",
    "        data = json.load(f)\n",
    "    return (data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a0651a6d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[{'LOWER': 'aluminium', 'POS': 'NOUN'}], [{'LOWER': 'cattle', 'POS': 'NOUN'}], [{'LOWER': 'cobalt', 'POS': 'NOUN'}], [{'LOWER': 'cocoa', 'POS': 'NOUN'}], [{'LOWER': 'coffee', 'POS': 'NOUN'}], [{'LOWER': 'copper', 'POS': 'NOUN'}], [{'LOWER': 'corn', 'POS': 'NOUN'}], [{'LOWER': 'cotton', 'POS': 'NOUN'}], [{'LOWER': 'crude', 'POS': 'ADJ'}, {'LOWER': 'oil', 'POS': 'NOUN'}], [{'LOWER': 'gold', 'POS': 'NOUN'}], [{'LOWER': 'iron', 'POS': 'NOUN'}, {'LOWER': 'ore', 'POS': 'NOUN'}], [{'LOWER': 'lithium', 'POS': 'NOUN'}], [{'LOWER': 'natural', 'POS': 'ADJ'}, {'LOWER': 'gas', 'POS': 'NOUN'}], [{'LOWER': 'palm', 'POS': 'NOUN'}, {'LOWER': 'oil', 'POS': 'NOUN'}], [{'LOWER': 'poultry', 'POS': 'NOUN'}], [{'LOWER': 'rice', 'POS': 'NOUN'}], [{'LOWER': 'silver', 'POS': 'NOUN'}], [{'LOWER': 'sugar', 'POS': 'NOUN'}], [{'LOWER': 'wheat', 'POS': 'NOUN'}], [{'LOWER': 'zinc', 'POS': 'NOUN'}]]\n"
     ]
    }
   ],
   "source": [
    "commodities = load_data(\"commodities-patterns.json\")\n",
    "print(commodities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "85d63e64",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_patterns(file, type):\n",
    "    data = load_data(file)\n",
    "    labelled_patterns = []\n",
    "    for pattern in data:\n",
    "        labelled_pattern = {\n",
    "            \"label\": type,\n",
    "            \"pattern\": pattern\n",
    "        }\n",
    "        labelled_patterns.append(labelled_pattern)\n",
    "    return (labelled_patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8633b1d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_rules(patterns):\n",
    "    nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "    nlp.remove_pipe(\"ner\")\n",
    "\n",
    "    ruler = nlp.add_pipe(\"entity_ruler\")\n",
    "\n",
    "    ruler.add_patterns(patterns)\n",
    "    \n",
    "    nlp.to_disk(\"commodities_ner_rules\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "88b4c548",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'COMMODITY', 'pattern': [{'LOWER': 'aluminium', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'cattle', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'cobalt', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'cocoa', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'coffee', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'copper', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'corn', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'cotton', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'crude', 'POS': 'ADJ'}, {'LOWER': 'oil', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'gold', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'iron', 'POS': 'NOUN'}, {'LOWER': 'ore', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'lithium', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'natural', 'POS': 'ADJ'}, {'LOWER': 'gas', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'palm', 'POS': 'NOUN'}, {'LOWER': 'oil', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'poultry', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'rice', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'silver', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'sugar', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'wheat', 'POS': 'NOUN'}]}, {'label': 'COMMODITY', 'pattern': [{'LOWER': 'zinc', 'POS': 'NOUN'}]}]\n"
     ]
    }
   ],
   "source": [
    "patterns = create_patterns(\"commodities-patterns.json\", \"COMMODITY\")\n",
    "print(patterns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "77a39404",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_rules(patterns)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}