main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import html5lib\n",
    "import requests\n",
    "import regex as re\n",
    "import bs4\n",
    "import random\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "data = pd.read_csv('urls/urls.csv')\n",
    "\n",
    "df = pd.DataFrame(columns = ['Url Index', 'Url', 'Title', 'Sub Title', 'Author', 'Author Url', 'Reading Time', 'Clap Count', 'Image Count/Sources', 'Article Text'])\n",
    "\n",
    "headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.42'}\n",
    "\n",
    "scrapped_links = []\n",
    "\n",
    "for i in range(data.shape[0]):\n",
    "    \n",
    "        cur_link_data = []\n",
    "        url_to_scrap = data.iloc[i]['url']\n",
    "        \n",
    "        try:\n",
    "        \n",
    "            r = requests.get(url_to_scrap, headers=headers, timeout = 20)\n",
    "            r_text = r.text\n",
    "            soup = BeautifulSoup(r_text, \"html5lib\")\n",
    "\n",
    "            try:\n",
    "                title = soup.find('h1').contents[0].text\n",
    "            except AttributeError:\n",
    "                title_raw = (soup.find('title')).text\n",
    "                title = title_raw[:title_raw.find(\"|\")]\n",
    "\n",
    "            if not title:\n",
    "                continue\n",
    "\n",
    "            try:\n",
    "                if \"Written by\" in soup.find('h2').contents[0].text:\n",
    "                    sub_title = \"None\"\n",
    "                else:\n",
    "                    sub_title = soup.find('h2').contents[0].text\n",
    "            except AttributeError:\n",
    "                sub_title = \"None\"\n",
    "\n",
    "            try:\n",
    "                author_name_raw = (soup.find(\"h2\", {\"class\": \"pw-author-name\"})).text\n",
    "                author_name = author_name_raw[11:]\n",
    "            except AttributeError:\n",
    "                author_name = \"None\"\n",
    "\n",
    "            try:\n",
    "                author_url_raw = (soup.find(\"h2\", {\"class\": \"pw-author-name\"})).parent['href']\n",
    "                author_url = \"https://medium.com\" + author_url_raw[:author_url_raw.find(\"?\")]\n",
    "            except AttributeError:\n",
    "                author_url = \"None\"\n",
    "\n",
    "            r_content = r.content.decode(\"utf-8\")\n",
    "            reading_time = r_content[r_content.find(\" min read\")-2:r_content.find(\" min read\")]\n",
    "            if reading_time[0] == '\"':\n",
    "                reading_time = reading_time[1:]\n",
    "\n",
    "            try:\n",
    "                clap_count = r_content.split(\"clapCount\\\":\")[1]\n",
    "            except IndexError:\n",
    "                clap_count = r_content.split(\"clapCount\\\":\")[0]\n",
    "            clap_count = clap_count[0:clap_count.find(\",\")]\n",
    "\n",
    "            reg = re.compile('(https:\\/\\/miro\\.medium\\.com\\/v2\\/resize:fit:)[\\d]+(\\/)(format:webp\\/|)(\\w|\\d|-|\\*)+(\\.jpeg|\\.png|\\.gif)')\n",
    "            img_tags = soup.find_all('picture')\n",
    "            img_srcs = str(len(img_tags))\n",
    "            for img in img_tags:\n",
    "                mos = reg.search(str(img))\n",
    "                try:\n",
    "                    img_srcs += \", \" + str(mos[0])\n",
    "                except TypeError:\n",
    "                    continue\n",
    "\n",
    "            text = \"\"\n",
    "            divTag = soup.find_all(\"div\", {\"class\": \"ch bg dx dy dz ea\"})\n",
    "            for tag in divTag:\n",
    "                tdTags = tag.find_all(\"p\")\n",
    "                for tag in tdTags:\n",
    "                    text += tag.text\n",
    "\n",
    "            if not text:\n",
    "                continue\n",
    "\n",
    "            df.loc[i] = [i, url_to_scrap, title, sub_title, author_name, author_url, reading_time, clap_count,img_srcs, text]\n",
    "            scrapped_links.append(url_to_scrap)\n",
    "            print(i)\n",
    "            time.sleep(1)\n",
    "        \n",
    "        except:\n",
    "            df.to_csv(\"data.csv\", encoding='utf-8')\n",
    "            time.sleep(5)\n",
    "\n",
    "df.to_csv(\"data.csv\", encoding='utf-8')\n",
    "print(\"-----Finished------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#------------Individual component scraping codes just for demonstration----------#\n",
    "print(soup.find('h1').contents[0].text) #Extract title\n",
    "print(soup.find('h2').contents[0].text) #Extract sub title\n",
    "print((soup.find(\"h2\", {\"class\": \"pw-author-name\"})).text) #Extract author name\n",
    "print((soup.find(\"h2\", {\"class\": \"pw-author-name\"})).parent['href']) #Extract author url\n",
    "print((soup.find(\"div\", {\"class\": \"speechify-ignore\"})).text) #Extract reading time\n",
    "#reading_time_raw = (soup.find(\"div\", {\"class\": \"speechify-ignore\"})).text\n",
    "#reg = re.compile('(\\d)+(?= min read)')\n",
    "#mo = reg.search(reading_time_raw)\n",
    "#reading_time = mo[0]\n",
    "divTag = soup.find_all(\"div\", {\"class\": \"ch bg dx dy dz ea\"}) #Extract article text\n",
    "#for tag in divTag:\n",
    "    #tdTags = tag.find_all(\"p\")\n",
    "    #for tag in tdTags:\n",
    "        #print(tag.text)\n",
    "reg = re.compile('(https:\\/\\/miro\\.medium\\.com\\/v2\\/resize:fit:)[\\d]+(\\/)(\\w|\\d|-|\\*)+(\\.jpeg|\\.png|\\.gif)') #Extract Images\n",
    "#reg = re.compile('(https:\\/\\/miro\\.medium\\.com\\/v2\\/resize:fit:)[\\d]+(\\/)(format:webp\\/|)(\\w|\\d|-|\\*)+(\\.jpeg|\\.png|\\.gif)')\n",
    "#img_tags = soup.find_all('picture')\n",
    "#for img in img_tags:\n",
    "    #mos = reg.search(str(img))\n",
    "    #print(mos[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}