wiki_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "62ba0758-6332-4eb0-b19d-6c1a51a8da57",
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4\n",
    "import regex as re\n",
    "import requests\n",
    "import unicodedata\n",
    "import time\n",
    "import concurrent.futures\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7481e0e5-58dc-4bb5-b7be-dc47b57d2d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "URL = 'https://en.wikipedia.org'\n",
    "html = requests.get(URL + '/wiki/List_of_ICD-9_codes').text\n",
    "soup = bs4.BeautifulSoup(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f57f6c74-86e8-4fb6-ae7b-a48e7051c5c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "icd9_pages = []\n",
    "for a in soup.find_all(href=re.compile('/wiki/List_of_ICD-9_codes_')):\n",
    "    icd9_pages.append(URL + a['href'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e64290de-61f7-4595-8d7b-5f41ca8fb6b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://en.wikipedia.org/wiki/List_of_ICD-9_codes_001%E2%80%93139:_infectious_and_parasitic_diseases',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_140%E2%80%93239:_neoplasms',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_240%E2%80%93279:_endocrine,_nutritional_and_metabolic_diseases,_and_immunity_disorders',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_280%E2%80%93289:_diseases_of_the_blood_and_blood-forming_organs',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_290%E2%80%93319:_mental_disorders',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_320%E2%80%93389:_diseases_of_the_nervous_system_and_sense_organs',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_390%E2%80%93459:_diseases_of_the_circulatory_system',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_460%E2%80%93519:_diseases_of_the_respiratory_system',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_520%E2%80%93579:_diseases_of_the_digestive_system',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_580%E2%80%93629:_diseases_of_the_genitourinary_system',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_630%E2%80%93679:_complications_of_pregnancy,_childbirth,_and_the_puerperium',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_680%E2%80%93709:_diseases_of_the_skin_and_subcutaneous_tissue',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_710%E2%80%93739:_diseases_of_the_musculoskeletal_system_and_connective_tissue',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_740%E2%80%93759:_congenital_anomalies',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_760%E2%80%93779:_certain_conditions_originating_in_the_perinatal_period',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_780%E2%80%93799:_symptoms,_signs,_and_ill-defined_conditions',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_800%E2%80%93999:_injury_and_poisoning',\n",
       " 'https://en.wikipedia.org/wiki/List_of_ICD-9_codes_E_and_V_codes:_external_causes_of_injury_and_supplemental_classification']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icd9_pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d37994f7-2f80-46f7-b96c-5d8a46060213",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_bullet(bullet, url=URL, recursive=False):\n",
    "    # icd9_code = bullet.find(href=re.compile('getICD9Code')).text\n",
    "    # article = bullet.find(href=re.compile('/wiki/'))\n",
    "    icd9_code = None\n",
    "    urls = []\n",
    "    if bullet.find('b', recursive=recursive):\n",
    "        hrefs = bullet.find('b', recursive=recursive).findAll(href=True, recursive=recursive)\n",
    "    else:\n",
    "        hrefs = bullet.findAll(href=True, recursive=recursive)\n",
    "    try:\n",
    "        icd9_code = hrefs[0].text\n",
    "        for href in hrefs[1:]:\n",
    "            if '/wiki/' in href['href']:\n",
    "                urls.append(url + href['href'])\n",
    "    except IndexError:\n",
    "        pass\n",
    "    return icd9_code, urls\n",
    "\n",
    "def parse_page(page):\n",
    "    data = []\n",
    "    soup = bs4.BeautifulSoup(page.text)\n",
    "    body = soup.find(class_='mw-parser-output')\n",
    "    lists = body.findChildren('ul', recursive=False)\n",
    "    for section in lists:\n",
    "        bullets = section.findChildren('li', recursive=False)\n",
    "        for bullet in bullets:\n",
    "            parent_code, parent_urls = parse_bullet(bullet)\n",
    "            if parent_code:\n",
    "                data.append((parent_code, parent_code, parent_urls, []))\n",
    "            if bullet.find('ul'):\n",
    "                sub_bullets = bullet.find('ul').findChildren('li', recursive=False)\n",
    "                for sub_bullet in sub_bullets:\n",
    "                    code, urls = parse_bullet(sub_bullet, recursive=True)\n",
    "                    if code:\n",
    "                        data.append((code, parent_code, urls, parent_urls))\n",
    "                    if sub_bullet.find('ul'):\n",
    "                        sub_bullets2 = sub_bullet.find('ul').findChildren('li', recursive=False)\n",
    "                        for sub_bullet2 in sub_bullets2:\n",
    "                            code2, urls2 = parse_bullet(sub_bullet2, recursive=True)\n",
    "                            if code2:\n",
    "                                data.append((code2, parent_code, urls2, parent_urls+urls))\n",
    "    return data\n",
    "\n",
    "def get_content(url):\n",
    "    r = requests.get(url)\n",
    "    soup = bs4.BeautifulSoup(r.text)\n",
    "    paragraphs = [str(paragraph.text) for paragraph in soup.find_all('p')]\n",
    "    headers = [str(paragraph.text) for paragraph in soup.find_all('span', attrs={'mw-headline'})]\n",
    "    text = [val for pair in zip(paragraphs, headers) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "    text = text.replace('\\n', '')\n",
    "    text = unicodedata.normalize('NFKD', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1dc00815-1cd9-4bc0-b3a1-4316cc7a3790",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_001%E2%80%93139:_infectious_and_parasitic_diseases\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_140%E2%80%93239:_neoplasms\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_240%E2%80%93279:_endocrine,_nutritional_and_metabolic_diseases,_and_immunity_disorders\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_280%E2%80%93289:_diseases_of_the_blood_and_blood-forming_organs\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_290%E2%80%93319:_mental_disorders\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_320%E2%80%93389:_diseases_of_the_nervous_system_and_sense_organs\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_390%E2%80%93459:_diseases_of_the_circulatory_system\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_460%E2%80%93519:_diseases_of_the_respiratory_system\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_520%E2%80%93579:_diseases_of_the_digestive_system\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_580%E2%80%93629:_diseases_of_the_genitourinary_system\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_630%E2%80%93679:_complications_of_pregnancy,_childbirth,_and_the_puerperium\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_680%E2%80%93709:_diseases_of_the_skin_and_subcutaneous_tissue\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_710%E2%80%93739:_diseases_of_the_musculoskeletal_system_and_connective_tissue\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_740%E2%80%93759:_congenital_anomalies\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_760%E2%80%93779:_certain_conditions_originating_in_the_perinatal_period\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_780%E2%80%93799:_symptoms,_signs,_and_ill-defined_conditions\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_800%E2%80%93999:_injury_and_poisoning\n",
      "https://en.wikipedia.org/wiki/List_of_ICD-9_codes_E_and_V_codes:_external_causes_of_injury_and_supplemental_classification\n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "for url in icd9_pages:\n",
    "    print(url)\n",
    "    page = requests.get(url)\n",
    "    part = parse_page(page)\n",
    "    data += part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f084121a-cd7f-4881-9848-0dec9744003d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text(url):\n",
    "    r = requests.get(url)\n",
    "    soup = bs4.BeautifulSoup(r.text)\n",
    "    paragraphs = [str(paragraph.text) for paragraph in soup.find_all('p')]\n",
    "    headers = [str(paragraph.text) for paragraph in soup.find_all('span', attrs={'mw-headline'})]\n",
    "    text = [val for pair in zip(paragraphs, headers) for val in pair]\n",
    "    text = ' '.join(text)\n",
    "    text = re.sub(r\"\\[.*?\\]+\", '', text)\n",
    "    text = text.replace('\\n', '')\n",
    "    text = unicodedata.normalize('NFKD', text)\n",
    "    return text, r.status_code\n",
    "\n",
    "def get_content(url, texts, statuses):\n",
    "    if not texts.get(url):\n",
    "        text, status_code = get_text(url)\n",
    "        texts[url] = text\n",
    "        if status_code != 200:\n",
    "            statuses.append(url)\n",
    "        time.sleep(0.25)\n",
    "        \n",
    "def get_contents(urls, texts):\n",
    "    statuses = []\n",
    "    threads = min(MAX_THREADS, len(urls))\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:\n",
    "        executor.map(lambda url: get_content(url, texts, statuses), urls)\n",
    "    return statuses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "db54111b-44b6-4742-8208-e16a1d97dab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_urls = set()\n",
    "for code, parent_code, urls, parent_urls in data:\n",
    "    for url in urls:\n",
    "        all_urls.add(url)\n",
    "all_urls = list(all_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1fdac506-ef31-4c8a-84df-5df9fce29957",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2467"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2a644f3a-dd10-4654-b68c-d5399e22ec16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "MAX_THREADS = 30\n",
    "texts = {}\n",
    "statuses = get_contents(all_urls, texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c5c09a5a-52e4-4118-b659-e961e2e45bca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def all_urls(x):\n",
    "    urls = set()\n",
    "    x.apply(lambda lst: [urls.add(itm) for itm in lst])\n",
    "    return list(urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "53dcc573-4f55-4d3c-925c-fe2756449bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data, columns=['code', 'parent_code', 'urls', 'parent_urls'])\n",
    "basic_codes = df.groupby('code').apply(lambda x: all_urls(x['urls'])).reset_index()\n",
    "basic_codes = basic_codes.rename({0: 'urls'}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "28eb2e72-113a-4760-84c5-47cbf4729984",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_codes(article):\n",
    "    out = set()\n",
    "    for _, code, urls in basic_codes.itertuples():\n",
    "        if article in urls:\n",
    "            out.add(code[:3])\n",
    "    return list(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9b748549",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "878"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_codes = set()\n",
    "with open(\"wiki_2\", 'wb') as f:\n",
    "    for article, text in texts.items():\n",
    "        codes = get_all_codes(article)\n",
    "        all_codes.update(codes)\n",
    "        codes = ' '.join([f'd_{code}' for code in codes])\n",
    "        f.write(f'XXXdiseaseXXX  {codes}\\n'.encode('utf8'))\n",
    "        f.write(text.encode('utf8'))\n",
    "        f.write('\\nXXXendXXX\\n'.encode('utf8'))\n",
    "len(all_codes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}