01.download-pubmed.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download XML efetch records for PubMed IDs in the specified journals\n",
    "\n",
    "Uses functions from `pubmedpy` rather than the `eutils` Python package, which [does not support](https://github.com/biocommons/eutils/issues/124/) pagination."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import lzma\n",
    "import tqdm\n",
    "\n",
    "from pubmedpy.eutilities import esearch_query, download_pubmed_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1f99c04df964f4cba711aa6403cd934",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=179656.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "179,656 articles for English computational biology\n",
      "CPU times: user 1.07 s, sys: 55.3 ms, total: 1.13 s\n",
      "Wall time: 14.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# for journal in journals:\n",
    "payload = {\n",
    "    'db': 'pubmed',\n",
    "    'term': '\"journal article\"[pt] AND 1993:2019[dp] AND \"computational biology\"[MeSH Terms] AND English[Language]',\n",
    "}\n",
    "pubmed_ids = esearch_query(payload, tqdm=tqdm.tqdm_notebook)\n",
    "pubmed_ids = sorted(map(int, pubmed_ids))\n",
    "print(f'{len(pubmed_ids):,} articles for English computational biology')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c9f1f4116e540bfb5424ffdbff2791c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=179656.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 2min 45s, sys: 1.85 s, total: 2min 47s\n",
      "Wall time: 9min 25s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path = pathlib.Path('data/pubmed/esummary/compbio-english.xml.xz')\n",
    "path.parent.mkdir(parents=True, exist_ok=True)\n",
    "with lzma.open(path, 'wt') as write_file:\n",
    "    download_pubmed_ids(\n",
    "        pubmed_ids, write_file, endpoint='esummary',\n",
    "        retmax=200, retmin=50, sleep=0, error_sleep=1,\n",
    "        tqdm=tqdm.tqdm_notebook,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c75a567cee04fdf8b51091de0d28786",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=179656.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 12min 30s, sys: 6.53 s, total: 12min 37s\n",
      "Wall time: 42min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "path = pathlib.Path('data/pubmed/efetch/compbio-english.xml.xz')\n",
    "path.parent.mkdir(parents=True, exist_ok=True)\n",
    "with lzma.open(path, 'wt') as write_file:\n",
    "    download_pubmed_ids(\n",
    "        pubmed_ids, write_file, endpoint='efetch',\n",
    "        retmax=200, retmin=50, sleep=0, error_sleep=1,\n",
    "        tqdm=tqdm.tqdm_notebook,\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}