1_get_covers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import glob\n",
    "import os\n",
    "from PIL import Image\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_URL = \"https://naruto.fandom.com\"\n",
    "volume_list_url = \"/wiki/List_of_Volumes\"\n",
    "res = requests.get(BASE_URL + volume_list_url)\n",
    "soup = BeautifulSoup(res.text, \"html.parser\")\n",
    "tables = soup.find_all(\"table\", {\"class\": \"wikitable\"})\n",
    "# The main Naruto series has 72 volumes, published in two parts of 27 and 45 volumes, respectively. These volumes are in the first two tables on the website.\n",
    "volumes = tables[0].find_all(\"div\", class_=\"floatright\") + tables[1].find_all(\n",
    "    \"div\", class_=\"floatright\"\n",
    ")\n",
    "volume_urls = [volume.find(\"a\").get(\"href\") for volume in volumes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "72it [00:21,  3.31it/s]\n"
     ]
    }
   ],
   "source": [
    "volumes = {}\n",
    "\n",
    "for volume, volume_url in tqdm(enumerate(volume_urls)):\n",
    "    res = requests.get(BASE_URL + volume_url)\n",
    "    soup = BeautifulSoup(res.text, \"html.parser\")\n",
    "    infobox = soup.find(\"table\", class_=\"infobox\")\n",
    "    volumes[volume] = {\n",
    "        \"name\": infobox.find(\"th\", class_=\"mainheader\").find(\"i\").text,\n",
    "        \"number\": int(\n",
    "            infobox.find(\"th\", class_=\"\", string=re.compile(r\"Volume\"))\n",
    "            .find_next_sibling(\"td\")\n",
    "            .find(string=True)\n",
    "            .strip()\n",
    "        ),\n",
    "        \"release_date\": (\n",
    "            infobox.find(\"th\", class_=\"\", string=re.compile(r\"English\"))\n",
    "            .find_next_sibling(\"td\")\n",
    "            .find(string=True)\n",
    "            .strip()\n",
    "        ),\n",
    "        \"cover_url\": infobox.find(\"a\", class_=\"image\").get(\"href\"),\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "volumes_df = pd.DataFrame.from_dict(volumes, orient=\"index\").set_index(\"number\")\n",
    "volumes_df[\"release_date\"] = pd.to_datetime(volumes_df[\"release_date\"])\n",
    "volumes_df.to_csv(\"data/volumes.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download all the covers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "72it [00:20,  3.43it/s]\n"
     ]
    }
   ],
   "source": [
    "for i, cover_url in tqdm(enumerate(volumes_df.cover_url.values)):\n",
    "    res = requests.get(cover_url)\n",
    "    with open(\"./data/covers/{}.webp\".format(i + 1), \"wb\") as f:\n",
    "        f.write(res.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 72/72 [00:00<00:00, 7089.30it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[((761, 1200), 10),\n",
       " ((762, 1200), 8),\n",
       " ((760, 1200), 5),\n",
       " ((764, 1200), 5),\n",
       " ((758, 1200), 4),\n",
       " ((763, 1200), 4),\n",
       " ((767, 1200), 3),\n",
       " ((765, 1200), 3),\n",
       " ((756, 1194), 3),\n",
       " ((759, 1200), 2),\n",
       " ((768, 1200), 2),\n",
       " ((757, 1200), 2),\n",
       " ((744, 1194), 2),\n",
       " ((750, 1194), 2),\n",
       " ((752, 1200), 1),\n",
       " ((753, 1200), 1),\n",
       " ((754, 1200), 1),\n",
       " ((782, 1200), 1),\n",
       " ((1041, 1600), 1),\n",
       " ((777, 1200), 1),\n",
       " ((768, 1194), 1),\n",
       " ((769, 1200), 1),\n",
       " ((770, 1200), 1),\n",
       " ((766, 1200), 1),\n",
       " ((1189, 1800), 1),\n",
       " ((1379, 2049), 1),\n",
       " ((683, 1037), 1),\n",
       " ((686, 1040), 1),\n",
       " ((1341, 2060), 1),\n",
       " ((786, 1188), 1),\n",
       " ((768, 1206), 1)]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sizes = {}\n",
    "for file in tqdm(glob.glob(\"./data/covers/*.jpg\")):\n",
    "    img = Image.open(file)\n",
    "    if img.size not in sizes:\n",
    "        sizes[img.size] = 1\n",
    "    else:\n",
    "        sizes[img.size] += 1\n",
    "# Sort by most common size\n",
    "sizes = sorted(sizes.items(), key=lambda x: x[1], reverse=True)\n",
    "sizes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Most of the images seem to be around (760, 1200) in size. So lets resize all the images to be that size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 72/72 [00:06<00:00, 11.37it/s]\n"
     ]
    }
   ],
   "source": [
    "images = glob.glob(\"data/covers/*\")\n",
    "for image in tqdm(images):\n",
    "    img = Image.open(image)\n",
    "    img = img.resize((760, 1200))\n",
    "    img.save(\"data/small_covers/\" + os.path.basename(image))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "hp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}