Web Scraper for Paleric Blog.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "819f16ca",
   "metadata": {},
   "source": [
    "# Web Scraper for Paleric Blog"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd700dd3",
   "metadata": {},
   "source": [
    "This is meant for educational purposes, to scrape and export the blog posts from the Chamorro blog written by Pale' Eric Forbes and compile the texts into an e-book for easier access and reference for Chamorro language learners."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53dd6082",
   "metadata": {},
   "source": [
    "**Name:** Schyuler Lujan <br>\n",
    "**Date Started:** 4-Nov-2024 <br>\n",
    "**Date Completed:** 5-Nov-2024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f93c35c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "from bs4 import BeautifulSoup\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc882cba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_older_urls(homepage):\n",
    "    \"\"\"\n",
    "    Get all the Older Posts links so we can navigate through the entire blog and get the links for each post\n",
    "    \"\"\"\n",
    "    # Request the homepage and parse the HTML\n",
    "    page = requests.get(homepage)\n",
    "    soup = BeautifulSoup(page.content, \"html.parser\")\n",
    "    \n",
    "    # Get the Older Posts link on the homepage; this is our starting point\n",
    "    older_urls = [homepage] # Initialize list for storing all links\n",
    "    older_anchors = soup.find_all('a', class_=\"blog-pager-older-link\")\n",
    "    \n",
    "    # Using the first Older Posts link, navigate to each subsequent Older Posts page until end of blog\n",
    "    while len(older_anchors) > 0:\n",
    "        older_urls.extend([anchor.get(\"href\") for anchor in older_anchors])\n",
    "        next_page = older_urls[-1] # Get the most recent URL\n",
    "        page = requests.get(next_page) # Navigate to the next page\n",
    "        soup = BeautifulSoup(page.content, \"html.parser\")\n",
    "        older_anchors = soup.find_all(\"a\", class_=\"blog-pager-older-link\")\n",
    "        \n",
    "    return older_urls    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ca8386b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_post_urls(navigation_urls):\n",
    "    \"\"\"\n",
    "    We will get all the urls for each individual post and save them to a list\n",
    "    To do this, we will iterate through all of the Older Posts urls\n",
    "    On each page, we will find all h3 tags with class_=\"post-title entry-title\" and extract the url\n",
    "    \"\"\"\n",
    "    post_urls = [] # Initialize list\n",
    "    \n",
    "    # On each page, get the URLS for each blog post\n",
    "    for url in navigation_urls:\n",
    "        page = requests.get(url)\n",
    "        soup = BeautifulSoup(page.content, \"html.parser\")\n",
    "        post_anchors = soup.find_all(\"h3\", class_=\"post-title entry-title\")\n",
    "        post_urls.extend([anchor.find(\"a\").get(\"href\") for anchor in post_anchors])\n",
    "    \n",
    "    return post_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbc6c614",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text(post_urls, blog_title):\n",
    "    \"\"\"\n",
    "    We will extract the actual blog content, format it nicely and export it to a single html file\n",
    "    We will iterate through post_urls to navigate to each blog post, then do the following:\n",
    "    - Get the post title -> find h3 class_=\"post-title entry-title\"\n",
    "    - Get the post date -> find h2 class_=\"date-header\"\n",
    "    - Get the post content -> find h3 class_=\"post-body entry-content\"\n",
    "    - Remove all images to lighten the file -> find img tags\n",
    "    - Create an HTML structure for formatting, and append each post to this structure\n",
    "    \"\"\"\n",
    "    \n",
    "    # Initialize the HTML structure for formatting, reading the HTML as UTF-8 to process special characters\n",
    "    combined_html_content = f\"\"\"\n",
    "    <html>\n",
    "    <meta charset=\"UTF-8\"><title>{blog_title}</title></head>\n",
    "    <body>\n",
    "    \"\"\"\n",
    "    # Go to each blog post and get the title, date and content\n",
    "    for url in post_urls:\n",
    "        page = requests.get(url)\n",
    "        page.encoding = \"utf-8\"\n",
    "        soup = BeautifulSoup(page.text, \"html.parser\")\n",
    "        \n",
    "        # Extract blog post title and blog content\n",
    "        title = soup.find(\"h3\", class_=\"post-title entry-title\").get_text(strip=True)\n",
    "        date = soup.find(\"h2\", class_=\"date-header\").get_text(strip=True)\n",
    "        content = soup.find('div', class_=\"post-body entry-content\")\n",
    "        \n",
    "        # Remove images from content\n",
    "        for img in content.find_all('img'):\n",
    "            img.decompose()\n",
    "        \n",
    "        # Append the blog post title, date and content to the HTML structure\n",
    "        combined_html_content += f\"\"\"\n",
    "        <section>\n",
    "        <h1>{title}</h1>\n",
    "        {date}\n",
    "        {content.prettify()}\n",
    "        </section>\n",
    "        \"\"\"\n",
    "        \n",
    "    # Close the HTML structure\n",
    "    combined_html_content += f\"\"\"\n",
    "    </body>\n",
    "    </html>\n",
    "    \"\"\"\n",
    "    \n",
    "    return combined_html_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fb0aaed",
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage = 'https://paleric.blogspot.com/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f64fb26e",
   "metadata": {},
   "outputs": [],
   "source": [
    "navigation_urls = get_older_urls(homepage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94e83336",
   "metadata": {},
   "outputs": [],
   "source": [
    "post_urls = get_post_urls(navigation_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea8a96a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_post_urls = post_urls[:6] # For testing\n",
    "blog_title = \"Paleric Blog\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04de781c",
   "metadata": {},
   "outputs": [],
   "source": [
    "blog_content = get_text(post_urls, blog_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a82b4b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final export\n",
    "with open(\"palericblog.html\", \"w\", encoding=\"utf-8\") as file:\n",
    "    file.write(blog_content)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}