merge_csvs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f960f031",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "\n",
    "\n",
    "path = \"downloads/pubmed_meta\" # use your path\n",
    "all_files = glob.glob(path + \"/*.csv\")\n",
    "\n",
    "li = []\n",
    "\n",
    "for filename in all_files:\n",
    "    df = pd.read_csv(filename, index_col=None, header=0)\n",
    "    li.append(df)\n",
    "\n",
    "frame = pd.concat(li, axis=0, ignore_index=True)\n",
    "frame.to_csv(\"downloads/concated.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cc07ba0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: biopython in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.79)\n",
      "Requirement already satisfied: spacy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (3.2.1)\n",
      "\u001b[31mERROR: Could not find a version that satisfies the requirement en_core_web_md (from versions: none)\u001b[0m\n",
      "\u001b[31mERROR: No matching distribution found for en_core_web_md\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install biopython spacy en_core_web_md\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f5df578",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nbdev\n",
      "  Using cached nbdev-1.1.23-py3-none-any.whl (46 kB)\n",
      "Collecting nbconvert<6\n",
      "  Downloading nbconvert-5.6.1-py2.py3-none-any.whl (455 kB)\n",
      "     |████████████████████████████████| 455 kB 28.7 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: jupyter-client<7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (6.1.11)\n",
      "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (21.3.1)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (21.3)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.4.1)\n",
      "Collecting fastrelease\n",
      "  Using cached fastrelease-0.1.12-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: nbformat>=4.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.1.2)\n",
      "Requirement already satisfied: jupyter in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (1.0.0)\n",
      "Requirement already satisfied: ipykernel in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.5.0)\n",
      "Collecting fastcore>=1.3.21\n",
      "  Using cached fastcore-1.3.27-py3-none-any.whl (56 kB)\n",
      "Collecting ghapi\n",
      "  Using cached ghapi-0.1.19-py3-none-any.whl (51 kB)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (4.7.1)\n",
      "Requirement already satisfied: tornado>=4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (6.1)\n",
      "Requirement already satisfied: traitlets in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (4.3.3)\n",
      "Requirement already satisfied: pyzmq>=13 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (22.0.3)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (2.8.1)\n",
      "Requirement already satisfied: jinja2>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (2.11.3)\n",
      "Requirement already satisfied: bleach in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (4.1.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (1.4.2)\n",
      "Requirement already satisfied: testpath in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.4.4)\n",
      "Requirement already satisfied: defusedxml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.6.0)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.3)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.8.4)\n",
      "Requirement already satisfied: pygments in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (2.8.0)\n",
      "Requirement already satisfied: ipython-genutils in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbformat>=4.4.0->nbdev) (0.2.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbformat>=4.4.0->nbdev) (3.2.0)\n",
      "Requirement already satisfied: ipython>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipykernel->nbdev) (7.16.1)\n",
      "Requirement already satisfied: qtconsole in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (5.0.2)\n",
      "Requirement already satisfied: jupyter-console in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (6.4.0)\n",
      "Requirement already satisfied: notebook in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (6.4.6)\n",
      "Requirement already satisfied: ipywidgets in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (7.6.3)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from packaging->nbdev) (2.4.7)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (3.0.5)\n",
      "Requirement already satisfied: setuptools>=18.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (49.6.0.post20210108)\n",
      "Requirement already satisfied: decorator in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (4.4.2)\n",
      "Requirement already satisfied: backcall in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.7.5)\n",
      "Requirement already satisfied: jedi>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.17.2)\n",
      "Requirement already satisfied: pexpect in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (4.8.0)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2>=2.4->nbconvert<6->nbdev) (1.1.1)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (0.17.3)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.7.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (20.3.0)\n",
      "Requirement already satisfied: six>=1.11.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (1.15.0)\n",
      "Requirement already satisfied: webencodings in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bleach->nbconvert<6->nbdev) (0.5.1)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipywidgets->jupyter->nbdev) (1.0.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipywidgets->jupyter->nbdev) (3.5.1)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (1.8.0)\n",
      "Requirement already satisfied: argon2-cffi in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (20.1.0)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (1.5.4)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (0.9.2)\n",
      "Requirement already satisfied: prometheus-client in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (0.9.0)\n",
      "Requirement already satisfied: qtpy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from qtconsole->jupyter->nbdev) (1.9.0)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jedi>=0.10->ipython>=5.0.0->ipykernel->nbdev) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=5.0.0->ipykernel->nbdev) (0.2.5)\n",
      "Requirement already satisfied: ptyprocess in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from terminado>=0.8.3->notebook->jupyter->nbdev) (0.7.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from argon2-cffi->notebook->jupyter->nbdev) (1.14.5)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.10.0.2)\n",
      "Requirement already satisfied: pycparser in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from cffi>=1.0.0->argon2-cffi->notebook->jupyter->nbdev) (2.20)\n",
      "Installing collected packages: nbconvert, fastcore, ghapi, fastrelease, nbdev\n",
      "  Attempting uninstall: nbconvert\n",
      "    Found existing installation: nbconvert 6.0.7\n",
      "    Uninstalling nbconvert-6.0.7:\n",
      "      Successfully uninstalled nbconvert-6.0.7\n",
      "Successfully installed fastcore-1.3.27 fastrelease-0.1.12 ghapi-0.1.19 nbconvert-5.6.1 nbdev-1.1.23\n",
      "Collecting fastai\n",
      "  Using cached fastai-2.5.3-py3-none-any.whl (189 kB)\n",
      "Collecting flatdict\n",
      "  Using cached flatdict-4.0.1.tar.gz (8.3 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: spacy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 3)) (3.2.1)\n",
      "Collecting bertopic\n",
      "  Downloading bertopic-0.9.3-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 767 kB/s             \n",
      "\u001b[?25hRequirement already satisfied: matplotlib in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (3.3.4)\n",
      "Requirement already satisfied: pydash in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 6)) (5.1.0)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 7)) (1.1.5)\n",
      "Requirement already satisfied: biopython in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 8)) (1.79)\n",
      "Collecting spacytextblob\n",
      "  Using cached spacytextblob-3.0.1-py3-none-any.whl (4.1 kB)\n",
      "Collecting sentence-transformers\n",
      "  Using cached sentence-transformers-2.1.0.tar.gz (78 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting top2vec\n",
      "  Using cached top2vec-1.0.26-py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (0.24.1)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (5.4.1)\n",
      "Collecting fastprogress>=0.2.4\n",
      "  Using cached fastprogress-1.0.0-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (2.26.0)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (21.3)\n",
      "Collecting torchvision>=0.8.2\n",
      "  Downloading torchvision-0.11.2-cp36-cp36m-manylinux1_x86_64.whl (23.3 MB)\n",
      "     |████████████████████████████████| 23.3 MB 93.9 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (1.5.3)\n",
      "Collecting fastdownload<2,>=0.0.5\n",
      "  Using cached fastdownload-0.0.5-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (21.3.1)\n",
      "Requirement already satisfied: pillow>6.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (8.4.0)\n",
      "Collecting torch<1.11,>=1.7.0\n",
      "  Downloading torch-1.10.1-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████                    | 327.9 MB 129.1 MB/s eta 0:00:05"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████        | 663.6 MB 120.3 MB/s eta 0:00:02"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████▋   | 789.5 MB 158.9 MB/s eta 0:00:01"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 881.9 MB 8.2 kB/s              \n",
      "\u001b[?25hRequirement already satisfied: fastcore<1.4,>=1.3.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (1.3.27)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.10.0.2)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (8.0.13)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.0.6)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (49.6.0.post20210108)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (4.62.3)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.19.5)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.8.2)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.3.0)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.4.0)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.0.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.9.0)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.11.3)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.0.6)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.4.2)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.7.5)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.0.6)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.6.1)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.0.8)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.0.6)\n",
      "Collecting hdbscan>=0.8.27\n",
      "  Using cached hdbscan-0.8.27.tar.gz (6.4 MB)\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting bertopic\n",
      "  Downloading bertopic-0.9.2-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 8.3 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.9.1-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.0 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.9.0-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.5 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.1-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 4.1 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.0-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 3.3 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.7.0-py2.py3-none-any.whl (40 kB)\n",
      "     |████████████████████████████████| 40 kB 11.5 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.6.0-py2.py3-none-any.whl (25 kB)\n",
      "Collecting umap-learn>=0.5.0\n",
      "  Using cached umap-learn-0.5.2.tar.gz (86 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.3.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.4.7)\n",
      "Requirement already satisfied: pytz>=2017.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 7)) (2021.1)\n",
      "Collecting textblob<0.16.0,>=0.15.3\n",
      "  Using cached textblob-0.15.3-py2.py3-none-any.whl (636 kB)\n",
      "Collecting transformers<5.0.0,>=4.6.0\n",
      "  Using cached transformers-4.15.0-py3-none-any.whl (3.4 MB)\n",
      "Collecting tokenizers>=0.10.3\n",
      "  Downloading tokenizers-0.11.4-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.8 MB)\n",
      "     |████████████████████████████████| 6.8 MB 85.7 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from sentence-transformers->-r requirements.txt (line 10)) (3.4.4)\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "     |████████████████████████████████| 1.2 MB 52.7 MB/s            \n",
      "\u001b[?25hCollecting huggingface-hub\n",
      "  Using cached huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
      "Collecting top2vec\n",
      "  Downloading top2vec-1.0.25-py3-none-any.whl (23 kB)\n",
      "  Downloading top2vec-1.0.24-py3-none-any.whl (22 kB)\n",
      "  Downloading top2vec-1.0.23-py3-none-any.whl (22 kB)\n",
      "  Downloading top2vec-1.0.22-py3-none-any.whl (22 kB)\n",
      "Collecting gensim\n",
      "  Downloading gensim-4.1.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (24.1 MB)\n",
      "     |████████████████████████████████| 24.1 MB 69.9 MB/s            \n",
      "\u001b[?25hCollecting wordcloud\n",
      "  Downloading wordcloud-1.8.1-cp36-cp36m-manylinux1_x86_64.whl (366 kB)\n",
      "     |████████████████████████████████| 366 kB 117.4 MB/s            \n",
      "\u001b[?25hCollecting tensorflow-text\n",
      "  Downloading tensorflow_text-2.6.0-cp36-cp36m-manylinux1_x86_64.whl (4.4 MB)\n",
      "     |████████████████████████████████| 4.4 MB 94.1 MB/s            \n",
      "\u001b[?25hCollecting tensorflow-hub\n",
      "  Using cached tensorflow_hub-0.12.0-py2.py3-none-any.whl (108 kB)\n",
      "Collecting tensorflow\n",
      "  Downloading tensorflow-2.6.2-cp36-cp36m-manylinux2010_x86_64.whl (458.3 MB)\n",
      "     |█████████████████████▉          | 313.2 MB 72.5 MB/s eta 0:00:03 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 458.3 MB 15 kB/s               \n",
      "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from catalogue<2.1.0,>=2.0.6->spacy->-r requirements.txt (line 3)) (3.4.0)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from cycler>=0.10->matplotlib->-r requirements.txt (line 5)) (1.15.0)\n",
      "Requirement already satisfied: cython>=0.27 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic->-r requirements.txt (line 4)) (0.29.22)\n",
      "Requirement already satisfied: joblib>=1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic->-r requirements.txt (line 4)) (1.0.1)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy->-r requirements.txt (line 3)) (0.8)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy->-r requirements.txt (line 3)) (5.2.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (1.26.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (2.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (3.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from scikit-learn->fastai->-r requirements.txt (line 1)) (2.1.0)\n",
      "Requirement already satisfied: contextvars<3,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.12->spacy->-r requirements.txt (line 3)) (2.4)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (3.0.12)\n",
      "Collecting tokenizers>=0.10.3\n",
      "  Downloading tokenizers-0.10.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
      "     |████████████████████████████████| 3.3 MB 58.0 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (2020.11.13)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (3.7.0)\n",
      "Collecting sacremoses\n",
      "  Using cached sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from typer<0.5.0,>=0.3.0->spacy->-r requirements.txt (line 3)) (7.1.2)\n",
      "Requirement already satisfied: numba>=0.49 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn>=0.5.0->bertopic->-r requirements.txt (line 4)) (0.52.0)\n",
      "Collecting pynndescent>=0.5\n",
      "  Using cached pynndescent-0.5.6.tar.gz (1.1 MB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2->spacy->-r requirements.txt (line 3)) (1.1.1)\n",
      "Collecting termcolor~=1.1.0\n",
      "  Using cached termcolor-1.1.0.tar.gz (3.9 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting grpcio<2.0,>=1.37.0\n",
      "  Downloading grpcio-1.43.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)\n",
      "     |████████████████████████████████| 4.1 MB 82.0 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: wrapt~=1.12.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (1.12.1)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (3.15.2)\n",
      "Collecting typing-extensions<4.0.0.0,>=3.7.4\n",
      "  Downloading typing_extensions-3.7.4.3-py3-none-any.whl (22 kB)\n",
      "Collecting flatbuffers~=1.12.0\n",
      "  Downloading flatbuffers-1.12-py2.py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: h5py~=3.1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (3.1.0)\n",
      "Collecting tensorboard<2.7,>=2.6.0\n",
      "  Downloading tensorboard-2.6.0-py3-none-any.whl (5.6 MB)\n",
      "     |████████████████████████████████| 5.6 MB 45.2 MB/s            \n",
      "\u001b[?25hCollecting tensorflow-estimator<2.7,>=2.6.0\n",
      "  Downloading tensorflow_estimator-2.6.0-py2.py3-none-any.whl (462 kB)\n",
      "     |████████████████████████████████| 462 kB 117.4 MB/s            \n",
      "\u001b[?25hCollecting absl-py~=0.10\n",
      "  Downloading absl_py-0.15.0-py3-none-any.whl (132 kB)\n",
      "     |████████████████████████████████| 132 kB 126.9 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: wheel~=0.35 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (0.36.2)\n",
      "Collecting astunparse~=1.6.3\n",
      "  Using cached astunparse-1.6.3-py2.py3-none-any.whl (12 kB)\n",
      "Collecting keras<2.7,>=2.6.0\n",
      "  Downloading keras-2.6.0-py2.py3-none-any.whl (1.3 MB)\n",
      "     |████████████████████████████████| 1.3 MB 56.4 MB/s            \n",
      "\u001b[?25hCollecting opt-einsum~=3.3.0\n",
      "  Using cached opt_einsum-3.3.0-py3-none-any.whl (65 kB)\n",
      "Collecting gast==0.4.0\n",
      "  Using cached gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
      "Collecting clang~=5.0\n",
      "  Downloading clang-5.0.tar.gz (30 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting keras-preprocessing~=1.1.2\n",
      "  Using cached Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42 kB)\n",
      "Requirement already satisfied: google-pasta~=0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (0.2.0)\n",
      "Requirement already satisfied: immutables>=0.9 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from contextvars<3,>=2.4->thinc<8.1.0,>=8.0.12->spacy->-r requirements.txt (line 3)) (0.15)\n",
      "Requirement already satisfied: cached-property in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from h5py~=3.1.0->tensorflow->top2vec->-r requirements.txt (line 12)) (1.5.1)\n",
      "Requirement already satisfied: llvmlite<0.36,>=0.35.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic->-r requirements.txt (line 4)) (0.35.0)\n",
      "Collecting markdown>=2.6.8\n",
      "  Using cached Markdown-3.3.6-py3-none-any.whl (97 kB)\n",
      "Collecting tensorboard-data-server<0.7.0,>=0.6.0\n",
      "  Using cached tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl (4.9 MB)\n",
      "Collecting tensorboard-plugin-wit>=1.6.0\n",
      "  Using cached tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)\n",
      "Collecting google-auth-oauthlib<0.5,>=0.4.1\n",
      "  Using cached google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (2.0.2)\n",
      "Collecting google-auth<2,>=1.6.3\n",
      "  Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n",
      "     |████████████████████████████████| 152 kB 122.7 MB/s            \n",
      "\u001b[?25hCollecting cachetools<5.0,>=2.0.0\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
      "Collecting pyasn1-modules>=0.2.1\n",
      "  Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (4.7.2)\n",
      "Collecting requests-oauthlib>=0.7.0\n",
      "  Using cached requests_oauthlib-1.3.0-py2.py3-none-any.whl (23 kB)\n",
      "Collecting importlib-metadata\n",
      "  Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (0.4.8)\n",
      "Collecting oauthlib>=3.0.0\n",
      "  Using cached oauthlib-3.1.1-py2.py3-none-any.whl (146 kB)\n",
      "Building wheels for collected packages: flatdict, sentence-transformers, hdbscan, umap-learn, clang, pynndescent, termcolor\n",
      "  Building wheel for flatdict (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for flatdict: filename=flatdict-4.0.1-py3-none-any.whl size=6931 sha256=6863eb7579a8d2f1926e3e988fcd1bd046862c5709bfe688fcdc19f810585748\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/8c/0b/50/16de22650bd1c28ef15caa790b1b61847aef6c1b57fcb5fe3a\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.1.0-py3-none-any.whl size=120999 sha256=3b040a7e7691f3a27e98f6d04232269e7ced22510401f6eda5c5a7d0fea09cb4\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/4e/6f/20/06e0c1e209742a37ce7a5a9aa4e420a3abd5081c65b4b34d0a\n",
      "  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for hdbscan: filename=hdbscan-0.8.27-cp36-cp36m-linux_x86_64.whl size=2667397 sha256=4b1acd89aa45fc8deeb67c23649cfab514884483d38b5ee84ec28bd9be4da891\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/70/52/fe/83ef5a0e879c48df9968d931d41046226e4cb19a166f0223d2\n",
      "  Building wheel for umap-learn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for umap-learn: filename=umap_learn-0.5.2-py3-none-any.whl size=82696 sha256=95c798c0f87005e2a75e25209cf4bd58a0cabff53af88f3f7718d64ca4b43b86\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/21/13/d2/20d08b12788c5c75e5e02cba181b9c8f1e4cdc8616aabf9209\n",
      "  Building wheel for clang (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for clang: filename=clang-5.0-py3-none-any.whl size=30702 sha256=789103012b88d87ac8d15c17ead17899898ce4b3de964ef54c44ffd256d10c44\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/22/4c/94/0583f60c9c5b6024ed64f290cb2d43b06bb4f75577dc3c93a7\n",
      "  Building wheel for pynndescent (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pynndescent: filename=pynndescent-0.5.6-py3-none-any.whl size=53931 sha256=517c0a42c507b644aa61a82dd3dfa524687bb06fbb285921be8abe22332bb273\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/84/0d/b8/fe1e8979b56095e4ea147085c997acf209f61e758765074f5a\n",
      "  Building wheel for termcolor (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for termcolor: filename=termcolor-1.1.0-py3-none-any.whl size=4829 sha256=f4c2618c0bdf17d56ffd6678b153bf2993491a3072f9bd50317427cd7d0ccf51\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/93/2a/eb/e58dbcbc963549ee4f065ff80a59f274cc7210b6eab962acdc\n",
      "Successfully built flatdict sentence-transformers hdbscan umap-learn clang pynndescent termcolor\n",
      "Installing collected packages: typing-extensions, pyasn1-modules, oauthlib, cachetools, requests-oauthlib, importlib-metadata, google-auth, tensorboard-plugin-wit, tensorboard-data-server, markdown, grpcio, google-auth-oauthlib, absl-py, torch, tokenizers, termcolor, tensorflow-estimator, tensorboard, sacremoses, pynndescent, opt-einsum, keras-preprocessing, keras, huggingface-hub, gast, flatbuffers, clang, astunparse, wordcloud, umap-learn, transformers, torchvision, tensorflow-hub, tensorflow, sentencepiece, hdbscan, gensim, fastprogress, top2vec, textblob, tensorflow-text, sentence-transformers, fastdownload, spacytextblob, flatdict, fastai, bertopic\n",
      "  Attempting uninstall: typing-extensions\n",
      "    Found existing installation: typing-extensions 3.10.0.2\n",
      "    Uninstalling typing-extensions-3.10.0.2:\n",
      "      Successfully uninstalled typing-extensions-3.10.0.2\n",
      "  Attempting uninstall: importlib-metadata\n",
      "    Found existing installation: importlib-metadata 3.7.0\n",
      "    Uninstalling importlib-metadata-3.7.0:\n",
      "      Successfully uninstalled importlib-metadata-3.7.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "pylint 2.12.2 requires typing-extensions>=3.10.0; python_version < \"3.10\", but you have typing-extensions 3.7.4.3 which is incompatible.\n",
      "astroid 2.9.0 requires typing-extensions>=3.10; python_version < \"3.10\", but you have typing-extensions 3.7.4.3 which is incompatible.\n",
      "aiobotocore 1.3.0 requires botocore<1.20.50,>=1.20.49, but you have botocore 1.23.25 which is incompatible.\u001b[0m\n",
      "Successfully installed absl-py-0.15.0 astunparse-1.6.3 bertopic-0.6.0 cachetools-4.2.4 clang-5.0 fastai-2.5.3 fastdownload-0.0.5 fastprogress-1.0.0 flatbuffers-1.12 flatdict-4.0.1 gast-0.4.0 gensim-4.1.2 google-auth-1.35.0 google-auth-oauthlib-0.4.6 grpcio-1.43.0 hdbscan-0.8.27 huggingface-hub-0.4.0 importlib-metadata-4.8.3 keras-2.6.0 keras-preprocessing-1.1.2 markdown-3.3.6 oauthlib-3.1.1 opt-einsum-3.3.0 pyasn1-modules-0.2.8 pynndescent-0.5.6 requests-oauthlib-1.3.0 sacremoses-0.0.47 sentence-transformers-2.1.0 sentencepiece-0.1.96 spacytextblob-3.0.1 tensorboard-2.6.0 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 tensorflow-2.6.2 tensorflow-estimator-2.6.0 tensorflow-hub-0.12.0 tensorflow-text-2.6.0 termcolor-1.1.0 textblob-0.15.3 tokenizers-0.10.3 top2vec-1.0.22 torch-1.10.1 torchvision-0.11.2 transformers-4.15.0 typing-extensions-3.7.4.3 umap-learn-0.5.2 wordcloud-1.8.1\n",
      "2022-01-27 02:52:06.315219: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-27 02:52:06.315250: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "Collecting en-core-web-md==3.2.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.2.0/en_core_web_md-3.2.0-py3-none-any.whl (45.7 MB)\n",
      "     |████████████████████████████████| 45.7 MB 118.1 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: spacy<3.3.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from en-core-web-md==3.2.0) (3.2.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.9.0)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4.2)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.19.5)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.0.8)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (21.3)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.0.6)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (49.6.0.post20210108)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.0.6)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (4.62.3)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.11.3)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (8.0.13)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.26.0)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.0.1)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.6.1)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.3.0)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.7.5)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.8.2)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.7.4.3)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.4.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (5.2.1)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.8)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.1)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.26.7)\n",
      "Requirement already satisfied: contextvars<3,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.12->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (7.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.1.1)\n",
      "Requirement already satisfied: immutables>=0.9 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from contextvars<3,>=2.4->thinc<8.1.0,>=8.0.12->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.15)\n",
      "Installing collected packages: en-core-web-md\n",
      "Successfully installed en-core-web-md-3.2.0\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_md')\n",
      "Converted 00_core.ipynb.\n",
      "Converted 00_utils.ipynb.\n",
      "Converted 01_colab_utils.ipynb.\n",
      "Converted 02_scattertext.ipynb.\n",
      "Converted 02_text_patterns.ipynb.\n",
      "Converted 03_classifier.ipynb.\n",
      "Converted 04_sentence_transformers.ipynb.\n",
      "Converted 98_embedding_projector.ipynb.\n",
      "Converted 99_pattern_CORE.ipynb.\n",
      "Converted 99_process_CORE_sample.ipynb.\n",
      "Converted 99_process_heffernan_ACL_dataset.ipynb.\n",
      "Converted 99_summarization.ipynb.\n",
      "Converted 99_text_analysis.ipynb.\n",
      "Converted archived_dataset_merge.ipynb.\n",
      "Converted flat_connecting.ipynb.\n",
      "Converted index.ipynb.\n",
      "Converted integrate_spreadsheet.ipynb.\n",
      "Converted merge_csvs.ipynb.\n",
      "Converted recursive_cluster.ipynb.\n",
      "Converted stem_only.ipynb.\n",
      "Converted topic_clustering.ipynb.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/02_text_patterns.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/archived_dataset_merge.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/03_classifier.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/integrate_spreadsheet.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/index.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/00_core.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/topic_clustering.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/04_sentence_transformers.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/00_utils.ipynb\n",
      "2022-01-27 02:52:26.609975: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-27 02:52:26.610025: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "2022-01-27 02:52:26.901835: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-27 02:52:26.901863: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/recursive_cluster.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/flat_connecting.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_pattern_CORE.ipynb\n"
     ]
    }
   ],
   "source": [
    "!sh setup.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d753f357",
   "metadata": {},
   "outputs": [],
   "source": [
    "from octoflow.core import get_pubmed_records, get_segment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d92c8fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"downloads/concated.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8c6a0bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96440"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"PMID\"] = df[\"PMID\"].astype(str)\n",
    "len(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caa4c3f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [1]\n",
    "a += [2]\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17aa661b",
   "metadata": {},
   "outputs": [],
   "source": [
    "records = []\n",
    "step = 10000\n",
    "for idx in range(step, 100001, step):\n",
    "    recs = get_pubmed_records(df[idx-step:idx][\"PMID\"])\n",
    "    records += recs[\"PubmedArticle\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc35caf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96440"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6536cdb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[StringElement('This is an updated version of the original Cochrane review first published in Issue 4, 2009. There is an ongoing debate about the indications for, and value of, adjuvant pelvic radiotherapy after radical surgery in women with early cervical cancer. Certain combinations of pathological risk factors are thought to represent sufficient risk for recurrence, that they justify the use of postoperative pelvic radiotherapy, though this has never been shown to improve overall survival, and use of more than one type of treatment (surgery and radiotherapy) increases the risks of side effects and complications.', attributes={'Label': 'BACKGROUND', 'NlmCategory': 'BACKGROUND'}),\n",
       " StringElement('To evaluate the effectiveness and safety of adjuvant therapies (radiotherapy, chemotherapy followed by radiotherapy, chemoradiation) after radical hysterectomy for early-stage cervical cancer (FIGO stages IB1, IB2 or IIA).', attributes={'Label': 'OBJECTIVES', 'NlmCategory': 'OBJECTIVE'}),\n",
       " StringElement('For the original review, we searched the Cochrane Central Register of Controlled Trials (CENTRAL), Issue 4, 2008. The Cochrane Gynaecological Cancer Group Trials Register, MEDLINE (January 1950 to November 2008), EMBASE (1950 to November 2008). We also searched registers of clinical trials, abstracts of scientific meetings, reference lists of included studies and contacted experts in the field. For this update, we extended the database searches to September 2011 and searched the MetaRegister for ongoing trials.', attributes={'Label': 'SEARCH METHODS', 'NlmCategory': 'METHODS'}),\n",
       " StringElement('Randomised controlled trials (RCTs) that compared adjuvant therapies (radiotherapy, chemotherapy followed by radiotherapy, or chemoradiation) with no radiotherapy or chemoradiation, in women with a confirmed histological diagnosis of early cervical cancer who had undergone radical hysterectomy and dissection of the pelvic lymph nodes.', attributes={'Label': 'SELECTION CRITERIA', 'NlmCategory': 'METHODS'}),\n",
       " StringElement('Two review authors independently abstracted data and assessed risk of bias. Information on grade 3 and 4 adverse events was collected from the trials. Results were pooled using random-effects meta-analyses.', attributes={'Label': 'DATA COLLECTION AND ANALYSIS', 'NlmCategory': 'METHODS'}),\n",
       " StringElement('Two RCTs, which compared adjuvant radiotherapy with no adjuvant radiotherapy, met the inclusion criteria; they randomised and assessed 397 women with stage IB cervical cancer. Meta-analysis of these two RCTs indicated no significant difference in survival at 5 years between women who received radiation and those who received no further treatment (risk ratio (RR) = 0.8; 95% confidence interval (CI) 0.3 to 2.4). However, women who received radiation had a significantly lower risk of disease progression at 5 years (RR 0.6; 95% CI 0.4 to 0.9).Although the risk of serious adverse events was consistently higher if women received radiotherapy rather than no further treatment, these increased risks were not statistically significant, probably because the rate of adverse events was low.', attributes={'Label': 'MAIN RESULTS', 'NlmCategory': 'RESULTS'}),\n",
       " StringElement('We found evidence, of moderate quality, that radiation decreases the risk of disease progression compared with no further treatment, but little evidence that it might improve overall survival, in stage IB cervical cancer. The evidence on serious adverse events was equivocal.', attributes={'Label': \"AUTHORS' CONCLUSIONS\", 'NlmCategory': 'CONCLUSIONS'})]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pydash import get\n",
    "get(records[1], \"MedlineCitation.Article.Abstract.AbstractText\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48dc3cbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"abstract\"] = [str(get(record, \"MedlineCitation.Article.Abstract.AbstractText.0\")) for record in records]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e976bd48",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"abstract\": \"Abstract\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49fb9146",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0e3218f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "\n",
    "alphabets= \"([A-Za-z])\"\n",
    "prefixes = \"(Mr|St|Mrs|Ms|Dr)[.]\"\n",
    "suffixes = \"(Inc|Ltd|Jr|Sr|Co)\"\n",
    "starters = \"(Mr|Mrs|Ms|Dr|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)\"\n",
    "acronyms = \"([A-Z][.][A-Z][.](?:[A-Z][.])?)\"\n",
    "websites = \"[.](com|net|org|io|gov)\"\n",
    "\n",
    "def split_into_sentences(text):\n",
    "    text = \" \" + text + \"  \"\n",
    "    text = text.replace(\"\\n\",\" \")\n",
    "    text = re.sub(prefixes,\"\\\\1<prd>\",text)\n",
    "    text = re.sub(websites,\"<prd>\\\\1\",text)\n",
    "    if \"Ph.D\" in text: text = text.replace(\"Ph.D.\",\"Ph<prd>D<prd>\")\n",
    "    text = re.sub(\"\\s\" + alphabets + \"[.] \",\" \\\\1<prd> \",text)\n",
    "    text = re.sub(acronyms+\" \"+starters,\"\\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\\\\3<prd>\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.] \"+starters,\" \\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.]\",\" \\\\1<prd>\",text)\n",
    "    text = re.sub(\" \" + alphabets + \"[.]\",\" \\\\1<prd>\",text)\n",
    "    if \"”\" in text: text = text.replace(\".”\",\"”.\")\n",
    "    if \"\\\"\" in text: text = text.replace(\".\\\"\",\"\\\".\")\n",
    "    if \"!\" in text: text = text.replace(\"!\\\"\",\"\\\"!\")\n",
    "    if \"?\" in text: text = text.replace(\"?\\\"\",\"\\\"?\")\n",
    "    text = text.replace(\".\",\".<stop>\")\n",
    "    text = text.replace(\"?\",\"?<stop>\")\n",
    "    text = text.replace(\"!\",\"!<stop>\")\n",
    "    text = text.replace(\"<prd>\",\".\")\n",
    "    sentences = text.split(\"<stop>\")\n",
    "    sentences = sentences[:-1]\n",
    "    sentences = [s.strip() for s in sentences]\n",
    "    return sentences\n",
    "\n",
    "def get_first(p):\n",
    "    return get(split_into_sentences(p), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28a26027",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"sentences\"] = df[\"Abstract\"].map(split_into_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ae00b4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"first_sent\"] = df[\"sentences\"].map(lambda s: get(s,0))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "845b057d",
   "metadata": {},
   "source": [
    "df.reset_index().to_csv(\"downloads/pm_abstracts_100k.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd28419f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Citation</th>\n",
       "      <th>First Author</th>\n",
       "      <th>Journal/Book</th>\n",
       "      <th>Publication Year</th>\n",
       "      <th>Create Date</th>\n",
       "      <th>PMCID</th>\n",
       "      <th>NIHMS ID</th>\n",
       "      <th>DOI</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>sentences</th>\n",
       "      <th>first_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>23706117</td>\n",
       "      <td>Screening for cervical cancer: a systematic re...</td>\n",
       "      <td>Peirson L, Fitzpatrick-Lewis D, Ciliska D, War...</td>\n",
       "      <td>Syst Rev. 2013 May 24;2:35. doi: 10.1186/2046-...</td>\n",
       "      <td>Peirson L</td>\n",
       "      <td>Syst Rev</td>\n",
       "      <td>2013</td>\n",
       "      <td>2013/05/28</td>\n",
       "      <td>PMC3681632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1186/2046-4053-2-35</td>\n",
       "      <td>The systematic review on which this paper is b...</td>\n",
       "      <td>[The systematic review on which this paper is ...</td>\n",
       "      <td>The systematic review on which this paper is b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>22592722</td>\n",
       "      <td>Radiotherapy and chemoradiation after surgery ...</td>\n",
       "      <td>Rogers L, Siu SS, Luesley D, Bryant A, Dickins...</td>\n",
       "      <td>Cochrane Database Syst Rev. 2012 May 16;5(5):C...</td>\n",
       "      <td>Rogers L</td>\n",
       "      <td>Cochrane Database Syst Rev</td>\n",
       "      <td>2012</td>\n",
       "      <td>2012/05/18</td>\n",
       "      <td>PMC4171000</td>\n",
       "      <td>EMS58087</td>\n",
       "      <td>10.1002/14651858.CD007583.pub3</td>\n",
       "      <td>This is an updated version of the original Coc...</td>\n",
       "      <td>[This is an updated version of the original Co...</td>\n",
       "      <td>This is an updated version of the original Coc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>28199532</td>\n",
       "      <td>Carcinogenicity of Human Papillomavirus (HPV) ...</td>\n",
       "      <td>Clifford GM, Tully S, Franceschi S.</td>\n",
       "      <td>Clin Infect Dis. 2017 May 1;64(9):1228-1235. d...</td>\n",
       "      <td>Clifford GM</td>\n",
       "      <td>Clin Infect Dis</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017/02/16</td>\n",
       "      <td>PMC5399941</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1093/cid/cix135</td>\n",
       "      <td>Data on the relative carcinogenic potential of...</td>\n",
       "      <td>[Data on the relative carcinogenic potential o...</td>\n",
       "      <td>Data on the relative carcinogenic potential of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>27598026</td>\n",
       "      <td>The role of Pap test screening against cervica...</td>\n",
       "      <td>Meggiolaro A, Unim B, Semyonov L, Miccoli S, M...</td>\n",
       "      <td>Clin Ter. 2016 Jul-Aug;167(4):124-39. doi: 10....</td>\n",
       "      <td>Meggiolaro A</td>\n",
       "      <td>Clin Ter</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016/09/07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.7417/CT.2016.1942</td>\n",
       "      <td>The first aim of this article is to quantify t...</td>\n",
       "      <td>[The first aim of this article is to quantify ...</td>\n",
       "      <td>The first aim of this article is to quantify t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>30824727</td>\n",
       "      <td>Prognostic Value of Preoperative Systemic Immu...</td>\n",
       "      <td>Huang H, Liu Q, Zhu L, Zhang Y, Lu X, Wu Y, Li...</td>\n",
       "      <td>Sci Rep. 2019 Mar 1;9(1):3284. doi: 10.1038/s4...</td>\n",
       "      <td>Huang H</td>\n",
       "      <td>Sci Rep</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019/03/03</td>\n",
       "      <td>PMC6397230</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/s41598-019-39150-0</td>\n",
       "      <td>The systemic immune-inflammation index (SII) b...</td>\n",
       "      <td>[The systemic immune-inflammation index (SII) ...</td>\n",
       "      <td>The systemic immune-inflammation index (SII) b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      PMID                                              Title  \\\n",
       "0           0  23706117  Screening for cervical cancer: a systematic re...   \n",
       "1           1  22592722  Radiotherapy and chemoradiation after surgery ...   \n",
       "2           2  28199532  Carcinogenicity of Human Papillomavirus (HPV) ...   \n",
       "3           3  27598026  The role of Pap test screening against cervica...   \n",
       "4           4  30824727  Prognostic Value of Preoperative Systemic Immu...   \n",
       "\n",
       "                                             Authors  \\\n",
       "0  Peirson L, Fitzpatrick-Lewis D, Ciliska D, War...   \n",
       "1  Rogers L, Siu SS, Luesley D, Bryant A, Dickins...   \n",
       "2                Clifford GM, Tully S, Franceschi S.   \n",
       "3  Meggiolaro A, Unim B, Semyonov L, Miccoli S, M...   \n",
       "4  Huang H, Liu Q, Zhu L, Zhang Y, Lu X, Wu Y, Li...   \n",
       "\n",
       "                                            Citation  First Author  \\\n",
       "0  Syst Rev. 2013 May 24;2:35. doi: 10.1186/2046-...     Peirson L   \n",
       "1  Cochrane Database Syst Rev. 2012 May 16;5(5):C...      Rogers L   \n",
       "2  Clin Infect Dis. 2017 May 1;64(9):1228-1235. d...   Clifford GM   \n",
       "3  Clin Ter. 2016 Jul-Aug;167(4):124-39. doi: 10....  Meggiolaro A   \n",
       "4  Sci Rep. 2019 Mar 1;9(1):3284. doi: 10.1038/s4...       Huang H   \n",
       "\n",
       "                 Journal/Book  Publication Year Create Date       PMCID  \\\n",
       "0                    Syst Rev              2013  2013/05/28  PMC3681632   \n",
       "1  Cochrane Database Syst Rev              2012  2012/05/18  PMC4171000   \n",
       "2             Clin Infect Dis              2017  2017/02/16  PMC5399941   \n",
       "3                    Clin Ter              2016  2016/09/07         NaN   \n",
       "4                     Sci Rep              2019  2019/03/03  PMC6397230   \n",
       "\n",
       "   NIHMS ID                             DOI  \\\n",
       "0       NaN          10.1186/2046-4053-2-35   \n",
       "1  EMS58087  10.1002/14651858.CD007583.pub3   \n",
       "2       NaN              10.1093/cid/cix135   \n",
       "3       NaN            10.7417/CT.2016.1942   \n",
       "4       NaN      10.1038/s41598-019-39150-0   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  The systematic review on which this paper is b...   \n",
       "1  This is an updated version of the original Coc...   \n",
       "2  Data on the relative carcinogenic potential of...   \n",
       "3  The first aim of this article is to quantify t...   \n",
       "4  The systemic immune-inflammation index (SII) b...   \n",
       "\n",
       "                                           sentences  \\\n",
       "0  [The systematic review on which this paper is ...   \n",
       "1  [This is an updated version of the original Co...   \n",
       "2  [Data on the relative carcinogenic potential o...   \n",
       "3  [The first aim of this article is to quantify ...   \n",
       "4  [The systemic immune-inflammation index (SII) ...   \n",
       "\n",
       "                                          first_sent  \n",
       "0  The systematic review on which this paper is b...  \n",
       "1  This is an updated version of the original Coc...  \n",
       "2  Data on the relative carcinogenic potential of...  \n",
       "3  The first aim of this article is to quantify t...  \n",
       "4  The systemic immune-inflammation index (SII) b...  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b81a689",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for index, r in [*df.iterrows()][:10]:\n",
    "#     #print(row)\n",
    "#     print(r[\"Title\"], r[\"abstract\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3e99284a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nbdev\n",
      "  Using cached nbdev-1.1.23-py3-none-any.whl (46 kB)\n",
      "Collecting nbconvert<6\n",
      "  Downloading nbconvert-5.6.1-py2.py3-none-any.whl (455 kB)\n",
      "     |████████████████████████████████| 455 kB 26.1 MB/s            \n",
      "\u001b[?25hCollecting fastrelease\n",
      "  Using cached fastrelease-0.1.12-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: jupyter in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (1.0.0)\n",
      "Requirement already satisfied: ipykernel in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.5.0)\n",
      "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (21.3.1)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.4.1)\n",
      "Requirement already satisfied: jupyter-client<7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (6.1.11)\n",
      "Collecting fastcore>=1.3.21\n",
      "  Using cached fastcore-1.3.27-py3-none-any.whl (56 kB)\n",
      "Requirement already satisfied: nbformat>=4.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (5.1.2)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbdev) (21.3)\n",
      "Collecting ghapi\n",
      "  Using cached ghapi-0.1.19-py3-none-any.whl (51 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (2.8.1)\n",
      "Requirement already satisfied: tornado>=4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (6.1)\n",
      "Requirement already satisfied: pyzmq>=13 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (22.0.3)\n",
      "Requirement already satisfied: jupyter-core>=4.6.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (4.7.1)\n",
      "Requirement already satisfied: traitlets in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter-client<7.0->nbdev) (4.3.3)\n",
      "Requirement already satisfied: bleach in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (4.1.0)\n",
      "Requirement already satisfied: testpath in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.4.4)\n",
      "Requirement already satisfied: jinja2>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (2.11.3)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.3)\n",
      "Requirement already satisfied: pygments in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (2.8.0)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.8.4)\n",
      "Requirement already satisfied: defusedxml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (0.6.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbconvert<6->nbdev) (1.4.2)\n",
      "Requirement already satisfied: ipython-genutils in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbformat>=4.4.0->nbdev) (0.2.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from nbformat>=4.4.0->nbdev) (3.2.0)\n",
      "Requirement already satisfied: ipython>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipykernel->nbdev) (7.16.1)\n",
      "Requirement already satisfied: notebook in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (6.4.6)\n",
      "Requirement already satisfied: qtconsole in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (5.0.2)\n",
      "Requirement already satisfied: ipywidgets in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (7.6.3)\n",
      "Requirement already satisfied: jupyter-console in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jupyter->nbdev) (6.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from packaging->nbdev) (2.4.7)\n",
      "Requirement already satisfied: pickleshare in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.7.5)\n",
      "Requirement already satisfied: backcall in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.2.0)\n",
      "Requirement already satisfied: pexpect in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (4.8.0)\n",
      "Requirement already satisfied: decorator in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (4.4.2)\n",
      "Requirement already satisfied: setuptools>=18.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (49.6.0.post20210108)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (3.0.5)\n",
      "Requirement already satisfied: jedi>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipython>=5.0.0->ipykernel->nbdev) (0.17.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2>=2.4->nbconvert<6->nbdev) (1.1.1)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.7.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (20.3.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (0.17.3)\n",
      "Requirement already satisfied: six>=1.11.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (1.15.0)\n",
      "Requirement already satisfied: webencodings in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from bleach->nbconvert<6->nbdev) (0.5.1)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipywidgets->jupyter->nbdev) (1.0.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from ipywidgets->jupyter->nbdev) (3.5.1)\n",
      "Requirement already satisfied: prometheus-client in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (0.9.0)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (1.8.0)\n",
      "Requirement already satisfied: argon2-cffi in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (20.1.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (0.9.2)\n",
      "Requirement already satisfied: nest-asyncio>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from notebook->jupyter->nbdev) (1.5.4)\n",
      "Requirement already satisfied: qtpy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from qtconsole->jupyter->nbdev) (1.9.0)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jedi>=0.10->ipython>=5.0.0->ipykernel->nbdev) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=5.0.0->ipykernel->nbdev) (0.2.5)\n",
      "Requirement already satisfied: ptyprocess in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from terminado>=0.8.3->notebook->jupyter->nbdev) (0.7.0)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from argon2-cffi->notebook->jupyter->nbdev) (1.14.5)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.10.0.2)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.4.0->nbdev) (3.4.0)\n",
      "Requirement already satisfied: pycparser in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from cffi>=1.0.0->argon2-cffi->notebook->jupyter->nbdev) (2.20)\n",
      "Installing collected packages: nbconvert, fastcore, ghapi, fastrelease, nbdev\n",
      "  Attempting uninstall: nbconvert\n",
      "    Found existing installation: nbconvert 6.0.7\n",
      "    Uninstalling nbconvert-6.0.7:\n",
      "      Successfully uninstalled nbconvert-6.0.7\n",
      "Successfully installed fastcore-1.3.27 fastrelease-0.1.12 ghapi-0.1.19 nbconvert-5.6.1 nbdev-1.1.23\n",
      "Collecting fastai\n",
      "  Using cached fastai-2.5.3-py3-none-any.whl (189 kB)\n",
      "Collecting flatdict\n",
      "  Using cached flatdict-4.0.1.tar.gz (8.3 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: spacy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 3)) (3.2.1)\n",
      "Collecting bertopic\n",
      "  Downloading bertopic-0.9.3-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 6.4 MB/s             \n",
      "\u001b[?25hRequirement already satisfied: matplotlib in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (3.3.4)\n",
      "Collecting pydash\n",
      "  Using cached pydash-5.1.0-py3-none-any.whl (84 kB)\n",
      "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 7)) (1.1.5)\n",
      "Requirement already satisfied: biopython in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from -r requirements.txt (line 8)) (1.79)\n",
      "Collecting spacytextblob\n",
      "  Using cached spacytextblob-3.0.1-py3-none-any.whl (4.1 kB)\n",
      "Collecting sentence-transformers\n",
      "  Using cached sentence-transformers-2.1.0.tar.gz (78 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting top2vec\n",
      "  Using cached top2vec-1.0.26-py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (21.3.1)\n",
      "Requirement already satisfied: pyyaml in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (5.4.1)\n",
      "Collecting fastprogress>=0.2.4\n",
      "  Using cached fastprogress-1.0.0-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: fastcore<1.4,>=1.3.22 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (1.3.27)\n",
      "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (0.24.1)\n",
      "Collecting torchvision>=0.8.2\n",
      "  Downloading torchvision-0.11.2-cp36-cp36m-manylinux1_x86_64.whl (23.3 MB)\n",
      "     |████████████████████████████████| 23.3 MB 95.7 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: requests in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (2.26.0)\n",
      "Collecting fastdownload<2,>=0.0.5\n",
      "  Using cached fastdownload-0.0.5-py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: scipy in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (1.5.3)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (21.3)\n",
      "Collecting torch<1.11,>=1.7.0\n",
      "  Downloading torch-1.10.2-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████▊                   | 349.4 MB 83.4 MB/s eta 0:00:07 "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |█████████████████████████▏      | 692.2 MB 139.3 MB/s eta 0:00:02"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 881.9 MB 7.6 kB/s              \n",
      "\u001b[?25hRequirement already satisfied: pillow>6.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from fastai->-r requirements.txt (line 1)) (8.4.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.0.8)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.4.0)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.6.1)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (4.62.3)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (8.0.13)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.0.6)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.3.0)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.0.6)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.4.2)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (3.10.0.2)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (49.6.0.post20210108)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.0.6)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.9.0)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.11.3)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (0.7.5)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.19.5)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.8.2)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (2.0.6)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy->-r requirements.txt (line 3)) (1.0.1)\n",
      "Collecting plotly<4.14.3,>=4.7.0\n",
      "  Downloading plotly-4.14.2-py2.py3-none-any.whl (13.2 MB)\n",
      "     |████████████████████████████████| 13.2 MB 28.3 MB/s            \n",
      "\u001b[?25hCollecting hdbscan>=0.8.27\n",
      "  Using cached hdbscan-0.8.27.tar.gz (6.4 MB)\n",
      "  Installing build dependencies ... \u001b[?25ldone\n",
      "\u001b[?25h  Getting requirements to build wheel ... \u001b[?25ldone\n",
      "\u001b[?25h  Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting umap-learn>=0.5.0\n",
      "  Using cached umap-learn-0.5.2.tar.gz (86 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting bertopic\n",
      "  Downloading bertopic-0.9.2-py2.py3-none-any.whl (57 kB)\n",
      "     |████████████████████████████████| 57 kB 8.1 MB/s             \n",
      "\u001b[?25h  Downloading bertopic-0.9.1-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.9 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.9.0-py2.py3-none-any.whl (55 kB)\n",
      "     |████████████████████████████████| 55 kB 7.4 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.1-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 574 kB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.8.0-py2.py3-none-any.whl (53 kB)\n",
      "     |████████████████████████████████| 53 kB 3.5 MB/s              \n",
      "\u001b[?25h  Downloading bertopic-0.7.0-py2.py3-none-any.whl (40 kB)\n",
      "     |████████████████████████████████| 40 kB 11.4 MB/s            \n",
      "\u001b[?25h  Downloading bertopic-0.6.0-py2.py3-none-any.whl (25 kB)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (1.3.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (2.4.7)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from matplotlib->-r requirements.txt (line 5)) (0.10.0)\n",
      "Requirement already satisfied: pytz>=2017.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas->-r requirements.txt (line 7)) (2021.1)\n",
      "Collecting textblob<0.16.0,>=0.15.3\n",
      "  Using cached textblob-0.15.3-py2.py3-none-any.whl (636 kB)\n",
      "Collecting transformers<5.0.0,>=4.6.0\n",
      "  Using cached transformers-4.16.0-py3-none-any.whl (3.5 MB)\n",
      "Collecting tokenizers>=0.10.3\n",
      "  Downloading tokenizers-0.11.4-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.8 MB)\n",
      "     |████████████████████████████████| 6.8 MB 60.4 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: nltk in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from sentence-transformers->-r requirements.txt (line 10)) (3.4.4)\n",
      "Collecting sentencepiece\n",
      "  Downloading sentencepiece-0.1.96-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
      "     |████████████████████████████████| 1.2 MB 86.2 MB/s            \n",
      "\u001b[?25hCollecting huggingface-hub\n",
      "  Using cached huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
      "Collecting gensim<4.0.0\n",
      "  Downloading gensim-3.8.3-cp36-cp36m-manylinux1_x86_64.whl (24.2 MB)\n",
      "     |████████████████████████████████| 24.2 MB 63.0 MB/s            \n",
      "\u001b[?25hCollecting wordcloud\n",
      "  Downloading wordcloud-1.8.1-cp36-cp36m-manylinux1_x86_64.whl (366 kB)\n",
      "     |████████████████████████████████| 366 kB 122.0 MB/s            \n",
      "\u001b[?25hCollecting top2vec\n",
      "  Downloading top2vec-1.0.25-py3-none-any.whl (23 kB)\n",
      "  Downloading top2vec-1.0.24-py3-none-any.whl (22 kB)\n",
      "  Downloading top2vec-1.0.23-py3-none-any.whl (22 kB)\n",
      "Collecting gensim\n",
      "  Downloading gensim-4.1.2-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (24.1 MB)\n",
      "     |████████████████████████████████| 24.1 MB 133.0 MB/s            \n",
      "\u001b[?25hCollecting top2vec\n",
      "  Downloading top2vec-1.0.22-py3-none-any.whl (22 kB)\n",
      "Collecting tensorflow-hub\n",
      "  Using cached tensorflow_hub-0.12.0-py2.py3-none-any.whl (108 kB)\n",
      "Collecting tensorflow-text\n",
      "  Downloading tensorflow_text-2.6.0-cp36-cp36m-manylinux1_x86_64.whl (4.4 MB)\n",
      "     |████████████████████████████████| 4.4 MB 132.1 MB/s            \n",
      "\u001b[?25hCollecting tensorflow\n",
      "  Downloading tensorflow-2.6.2-cp36-cp36m-manylinux2010_x86_64.whl (458.3 MB)\n",
      "     |█████████████████████▊          | 310.8 MB 116.1 MB/s eta 0:00:02"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 458.3 MB 6.6 kB/s              \n",
      "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from catalogue<2.1.0,>=2.0.6->spacy->-r requirements.txt (line 3)) (3.4.0)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from cycler>=0.10->matplotlib->-r requirements.txt (line 5)) (1.15.0)\n",
      "Requirement already satisfied: joblib>=1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic->-r requirements.txt (line 4)) (1.0.1)\n",
      "Requirement already satisfied: cython>=0.27 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from hdbscan>=0.8.27->bertopic->-r requirements.txt (line 4)) (0.29.22)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy->-r requirements.txt (line 3)) (0.8)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy->-r requirements.txt (line 3)) (5.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (2021.5.30)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (2.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (3.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests->fastai->-r requirements.txt (line 1)) (1.26.7)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from scikit-learn->fastai->-r requirements.txt (line 1)) (2.1.0)\n",
      "Requirement already satisfied: contextvars<3,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.12->spacy->-r requirements.txt (line 3)) (2.4)\n",
      "Collecting torch<1.11,>=1.7.0\n",
      "  Downloading torch-1.10.1-cp36-cp36m-manylinux1_x86_64.whl (881.9 MB)\n",
      "     |████████████▊                   | 350.4 MB 142.3 MB/s eta 0:00:04"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |██████████████████████████      | 716.2 MB 126.6 MB/s eta 0:00:02"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     |████████████████████████████████| 881.9 MB 4.3 kB/s              \n",
      "\u001b[?25hRequirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (3.7.0)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (3.0.12)\n",
      "Collecting sacremoses\n",
      "  Using cached sacremoses-0.0.47-py2.py3-none-any.whl (895 kB)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers->-r requirements.txt (line 10)) (2020.11.13)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from typer<0.5.0,>=0.3.0->spacy->-r requirements.txt (line 3)) (7.1.2)\n",
      "Requirement already satisfied: numba>=0.49 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from umap-learn>=0.5.0->bertopic->-r requirements.txt (line 4)) (0.52.0)\n",
      "Collecting pynndescent>=0.5\n",
      "  Using cached pynndescent-0.5.6.tar.gz (1.1 MB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2->spacy->-r requirements.txt (line 3)) (1.1.1)\n",
      "Collecting grpcio<2.0,>=1.37.0\n",
      "  Downloading grpcio-1.43.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)\n",
      "     |████████████████████████████████| 4.1 MB 87.2 MB/s            \n",
      "\u001b[?25hCollecting astunparse~=1.6.3\n",
      "  Using cached astunparse-1.6.3-py2.py3-none-any.whl (12 kB)\n",
      "Collecting keras-preprocessing~=1.1.2\n",
      "  Using cached Keras_Preprocessing-1.1.2-py2.py3-none-any.whl (42 kB)\n",
      "Requirement already satisfied: wheel~=0.35 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (0.36.2)\n",
      "Collecting gast==0.4.0\n",
      "  Using cached gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (1.12.1)\n",
      "Collecting opt-einsum~=3.3.0\n",
      "  Using cached opt_einsum-3.3.0-py3-none-any.whl (65 kB)\n",
      "Collecting flatbuffers~=1.12.0\n",
      "  Downloading flatbuffers-1.12-py2.py3-none-any.whl (15 kB)\n",
      "Requirement already satisfied: h5py~=3.1.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (3.1.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (3.15.2)\n",
      "Collecting typing-extensions<4.0.0.0,>=3.7.4\n",
      "  Downloading typing_extensions-3.7.4.3-py3-none-any.whl (22 kB)\n",
      "Collecting tensorflow-estimator<2.7,>=2.6.0\n",
      "  Downloading tensorflow_estimator-2.6.0-py2.py3-none-any.whl (462 kB)\n",
      "     |████████████████████████████████| 462 kB 121.2 MB/s            \n",
      "\u001b[?25hCollecting absl-py~=0.10\n",
      "  Downloading absl_py-0.15.0-py3-none-any.whl (132 kB)\n",
      "     |████████████████████████████████| 132 kB 123.8 MB/s            \n",
      "\u001b[?25hCollecting termcolor~=1.1.0\n",
      "  Using cached termcolor-1.1.0.tar.gz (3.9 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting keras<2.7,>=2.6.0\n",
      "  Downloading keras-2.6.0-py2.py3-none-any.whl (1.3 MB)\n",
      "     |████████████████████████████████| 1.3 MB 64.1 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: google-pasta~=0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorflow->top2vec->-r requirements.txt (line 12)) (0.2.0)\n",
      "Collecting clang~=5.0\n",
      "  Downloading clang-5.0.tar.gz (30 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting tensorboard<2.7,>=2.6.0\n",
      "  Downloading tensorboard-2.6.0-py3-none-any.whl (5.6 MB)\n",
      "     |████████████████████████████████| 5.6 MB 39.9 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: immutables>=0.9 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from contextvars<3,>=2.4->thinc<8.1.0,>=8.0.12->spacy->-r requirements.txt (line 3)) (0.15)\n",
      "Requirement already satisfied: cached-property in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from h5py~=3.1.0->tensorflow->top2vec->-r requirements.txt (line 12)) (1.5.1)\n",
      "Requirement already satisfied: llvmlite<0.36,>=0.35.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from numba>=0.49->umap-learn>=0.5.0->bertopic->-r requirements.txt (line 4)) (0.35.0)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (2.0.2)\n",
      "Collecting tensorboard-data-server<0.7.0,>=0.6.0\n",
      "  Using cached tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl (4.9 MB)\n",
      "Collecting google-auth<2,>=1.6.3\n",
      "  Downloading google_auth-1.35.0-py2.py3-none-any.whl (152 kB)\n",
      "     |████████████████████████████████| 152 kB 122.2 MB/s            \n",
      "\u001b[?25hCollecting tensorboard-plugin-wit>=1.6.0\n",
      "  Using cached tensorboard_plugin_wit-1.8.1-py3-none-any.whl (781 kB)\n",
      "Collecting google-auth-oauthlib<0.5,>=0.4.1\n",
      "  Using cached google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\n",
      "Collecting markdown>=2.6.8\n",
      "  Using cached Markdown-3.3.6-py3-none-any.whl (97 kB)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (4.7.2)\n",
      "Collecting cachetools<5.0,>=2.0.0\n",
      "  Downloading cachetools-4.2.4-py3-none-any.whl (10 kB)\n",
      "Collecting pyasn1-modules>=0.2.1\n",
      "  Using cached pyasn1_modules-0.2.8-py2.py3-none-any.whl (155 kB)\n",
      "Collecting requests-oauthlib>=0.7.0\n",
      "  Using cached requests_oauthlib-1.3.0-py2.py3-none-any.whl (23 kB)\n",
      "Collecting importlib-metadata\n",
      "  Downloading importlib_metadata-4.8.3-py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.7,>=2.6.0->tensorflow->top2vec->-r requirements.txt (line 12)) (0.4.8)\n",
      "Collecting oauthlib>=3.0.0\n",
      "  Using cached oauthlib-3.1.1-py2.py3-none-any.whl (146 kB)\n",
      "Building wheels for collected packages: flatdict, sentence-transformers, hdbscan, umap-learn, clang, pynndescent, termcolor\n",
      "  Building wheel for flatdict (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for flatdict: filename=flatdict-4.0.1-py3-none-any.whl size=6931 sha256=a4ee0641b697d7f78484365a329e527668cabff3e499dba6ea8f919d6a0c84c3\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/8c/0b/50/16de22650bd1c28ef15caa790b1b61847aef6c1b57fcb5fe3a\n",
      "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sentence-transformers: filename=sentence_transformers-2.1.0-py3-none-any.whl size=120999 sha256=df767ab285b47dc9c926836811a3b0096be56449754303e7f884b1096f47abb1\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/4e/6f/20/06e0c1e209742a37ce7a5a9aa4e420a3abd5081c65b4b34d0a\n",
      "  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for hdbscan: filename=hdbscan-0.8.27-cp36-cp36m-linux_x86_64.whl size=2667437 sha256=70976d226fc5a007f6758de8147bce556ef7323da251b901594ed3aabe46b21c\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/70/52/fe/83ef5a0e879c48df9968d931d41046226e4cb19a166f0223d2\n",
      "  Building wheel for umap-learn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for umap-learn: filename=umap_learn-0.5.2-py3-none-any.whl size=82696 sha256=7e2e49e15b93320a228592a029e765d3d02fe2d64e8a6f8e77cae35a2a570452\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/21/13/d2/20d08b12788c5c75e5e02cba181b9c8f1e4cdc8616aabf9209\n",
      "  Building wheel for clang (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for clang: filename=clang-5.0-py3-none-any.whl size=30702 sha256=19f6fc074e5a5cf0ec273c94f6ef386b93a62e8e6713e86fd046302d5dbc54ee\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/22/4c/94/0583f60c9c5b6024ed64f290cb2d43b06bb4f75577dc3c93a7\n",
      "  Building wheel for pynndescent (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pynndescent: filename=pynndescent-0.5.6-py3-none-any.whl size=53931 sha256=d3f015f23996a0a0c4b30e58cb2d9994e5fa707a0ad69ab308d68654344ff705\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/84/0d/b8/fe1e8979b56095e4ea147085c997acf209f61e758765074f5a\n",
      "  Building wheel for termcolor (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for termcolor: filename=termcolor-1.1.0-py3-none-any.whl size=4829 sha256=ba903d2e90acc43e26ad88bb33f066681c09e5dcdfc2841ffeba9967cee8d311\n",
      "  Stored in directory: /home/ec2-user/.cache/pip/wheels/93/2a/eb/e58dbcbc963549ee4f065ff80a59f274cc7210b6eab962acdc\n",
      "Successfully built flatdict sentence-transformers hdbscan umap-learn clang pynndescent termcolor\n",
      "Installing collected packages: typing-extensions, pyasn1-modules, oauthlib, cachetools, requests-oauthlib, importlib-metadata, google-auth, tensorboard-plugin-wit, tensorboard-data-server, markdown, grpcio, google-auth-oauthlib, absl-py, torch, tokenizers, termcolor, tensorflow-estimator, tensorboard, sacremoses, pynndescent, opt-einsum, keras-preprocessing, keras, huggingface-hub, gast, flatbuffers, clang, astunparse, wordcloud, umap-learn, transformers, torchvision, tensorflow-hub, tensorflow, sentencepiece, hdbscan, gensim, fastprogress, top2vec, textblob, tensorflow-text, sentence-transformers, fastdownload, spacytextblob, pydash, flatdict, fastai, bertopic\n",
      "  Attempting uninstall: typing-extensions\n",
      "    Found existing installation: typing-extensions 3.10.0.2\n",
      "    Uninstalling typing-extensions-3.10.0.2:\n",
      "      Successfully uninstalled typing-extensions-3.10.0.2\n",
      "  Attempting uninstall: importlib-metadata\n",
      "    Found existing installation: importlib-metadata 3.7.0\n",
      "    Uninstalling importlib-metadata-3.7.0:\n",
      "      Successfully uninstalled importlib-metadata-3.7.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "pylint 2.12.2 requires typing-extensions>=3.10.0; python_version < \"3.10\", but you have typing-extensions 3.7.4.3 which is incompatible.\n",
      "astroid 2.9.0 requires typing-extensions>=3.10; python_version < \"3.10\", but you have typing-extensions 3.7.4.3 which is incompatible.\n",
      "aiobotocore 1.3.0 requires botocore<1.20.50,>=1.20.49, but you have botocore 1.23.25 which is incompatible.\u001b[0m\n",
      "Successfully installed absl-py-0.15.0 astunparse-1.6.3 bertopic-0.6.0 cachetools-4.2.4 clang-5.0 fastai-2.5.3 fastdownload-0.0.5 fastprogress-1.0.0 flatbuffers-1.12 flatdict-4.0.1 gast-0.4.0 gensim-4.1.2 google-auth-1.35.0 google-auth-oauthlib-0.4.6 grpcio-1.43.0 hdbscan-0.8.27 huggingface-hub-0.4.0 importlib-metadata-4.8.3 keras-2.6.0 keras-preprocessing-1.1.2 markdown-3.3.6 oauthlib-3.1.1 opt-einsum-3.3.0 pyasn1-modules-0.2.8 pydash-5.1.0 pynndescent-0.5.6 requests-oauthlib-1.3.0 sacremoses-0.0.47 sentence-transformers-2.1.0 sentencepiece-0.1.96 spacytextblob-3.0.1 tensorboard-2.6.0 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 tensorflow-2.6.2 tensorflow-estimator-2.6.0 tensorflow-hub-0.12.0 tensorflow-text-2.6.0 termcolor-1.1.0 textblob-0.15.3 tokenizers-0.11.4 top2vec-1.0.22 torch-1.10.1 torchvision-0.11.2 transformers-4.16.0 typing-extensions-3.7.4.3 umap-learn-0.5.2 wordcloud-1.8.1\n",
      "2022-01-28 01:52:28.851571: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-28 01:52:28.851601: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "Collecting en-core-web-md==3.2.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.2.0/en_core_web_md-3.2.0-py3-none-any.whl (45.7 MB)\n",
      "     |████████████████████████████████| 45.7 MB 1.5 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: spacy<3.3.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from en-core-web-md==3.2.0) (3.2.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (21.3)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.11.3)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4.2)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.4.0)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (4.62.3)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.0.8)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.7.4.3)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (8.0.13)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.9.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.26.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.19.5)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (49.6.0.post20210108)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.6.1)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.0.6)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.8.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.0.6)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.3.0)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.0.1)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.7.5)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (5.2.1)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.26.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.0.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (3.1)\n",
      "Requirement already satisfied: contextvars<3,>=2.4 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.12->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (2.4)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (7.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (1.1.1)\n",
      "Requirement already satisfied: immutables>=0.9 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from contextvars<3,>=2.4->thinc<8.1.0,>=8.0.12->spacy<3.3.0,>=3.2.0->en-core-web-md==3.2.0) (0.15)\n",
      "Installing collected packages: en-core-web-md\n",
      "Successfully installed en-core-web-md-3.2.0\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_md')\n",
      "Converted 00_core.ipynb.\n",
      "Converted 00_utils.ipynb.\n",
      "Converted 01_colab_utils.ipynb.\n",
      "Converted 02_scattertext.ipynb.\n",
      "Converted 02_text_patterns.ipynb.\n",
      "Converted 03_classifier.ipynb.\n",
      "Converted 04_sentence_transformers.ipynb.\n",
      "Converted 98_embedding_projector.ipynb.\n",
      "Converted 99_pattern_CORE.ipynb.\n",
      "Converted 99_process_CORE_sample.ipynb.\n",
      "Converted 99_process_heffernan_ACL_dataset.ipynb.\n",
      "Converted 99_summarization.ipynb.\n",
      "Converted 99_text_analysis.ipynb.\n",
      "Converted archived_dataset_merge.ipynb.\n",
      "Converted flat_connecting.ipynb.\n",
      "Converted index.ipynb.\n",
      "Converted integrate_spreadsheet.ipynb.\n",
      "Converted merge_csvs.ipynb.\n",
      "Converted recursive_cluster.ipynb.\n",
      "Converted stem_only.ipynb.\n",
      "Converted topic_clustering.ipynb.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/02_text_patterns.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/archived_dataset_merge.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/03_classifier.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/integrate_spreadsheet.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/00_core.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/04_sentence_transformers.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/index.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/topic_clustering.ipynb\n",
      "2022-01-28 01:52:54.320146: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-28 01:52:54.320176: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/00_utils.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/recursive_cluster.ipynb\n",
      "2022-01-28 01:52:54.581825: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-28 01:52:54.581853: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/flat_connecting.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_pattern_CORE.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_text_analysis.ipynb\n",
      "2022-01-28 01:52:57.687192: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/usr/local/cuda-10.0/lib:/usr/local/cuda-10.0/efa/lib:/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/efa/lib64:/opt/amazon/openmpi/lib64:/usr/local/lib:/usr/lib:/lib:\n",
      "2022-01-28 01:52:57.687242: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_summarization.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/01_colab_utils.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/98_embedding_projector.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_process_heffernan_ACL_dataset.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/stem_only.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/99_process_CORE_sample.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/merge_csvs.ipynb\n",
      "converting: /home/ec2-user/SageMaker/octoflow/02_scattertext.ipynb\n",
      "Warning: Some of your Notebooks use the same title (['Title', 'Title', 'Title', 'Title', 'Finding (Problem Statement) Signal in Sentence Phrasing', 'Title', 'Title', 'Title', 'Pattern Sanity Check on CORE Abstracts', 'Title', 'Title', 'Title', 'Text Analysis ', 'Title', 'Closed Links in Eucledian Clustering', 'Integrate Labeled Dataset', 'Title', 'Recursive Clustering and Summarization', 'STEM SUBSET OF CORE', 'Clustering Sentences (non-hierachical)']).\n",
      "converting /home/ec2-user/SageMaker/octoflow/index.ipynb to README.md\n"
     ]
    }
   ],
   "source": [
    "!sh setup.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a7639181",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "from octoflow.core import replace_outof_vocab_words, vocab30k\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a8af3510",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"downloads/pm_abstracts_100k.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ce980078",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95796"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "50f96ea4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94678"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#df = df.dropna(subset=[\"labels\"])\n",
    "df[\"text\"] = df[\"first_sent\"] #backup\n",
    "\n",
    "df[\"text_length\"]= df[\"text\"].map(lambda t: len(t.split(\" \")))\n",
    "\n",
    "df = df[df[\"text_length\"] > 7]\n",
    "\n",
    "len(df)\n",
    "\n",
    "#df[\"labels\"] = df[\"labels\"].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c30be08",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"text\"] = df[\"text\"].map(lambda t: replace_outof_vocab_words(t, vocab30k))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f95dd39",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"text\"] = df[\"text\"].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8168c42b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"downloads/pm_abstracts_100k_masked.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0e72e1f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}