myanimelist_data_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import multiprocessing\n",
    "import concurrent.futures\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "def extract_features(a_soup):\n",
    "    # Init empty dict\n",
    "    a_dict = {}\n",
    "    req_features_list = ['anime_id', 'Title', 'Synonyms', 'Type', 'Episodes', 'Status', 'Aired', 'Premiered', 'Broadcast', 'Producers', 'Licensors', 'Studios', 'Source', 'Genres', 'Themes', 'Demographic', 'Duration', 'Rating', 'Score', 'Ranked', 'Popularity', 'Members', 'Favorites']\n",
    "    \n",
    "    # Fetch anime title & id\n",
    "    a_dict['Title'] = a_soup.find('h1', class_='title-name h1_bold_none').get_text()\n",
    "    a_dict['anime_id'] = a_soup.find('div', id='content').find('a')['href'].split(sep='/')[4]\n",
    "\n",
    "    # Fetch alternative titles, information & statistics\n",
    "    for div in a_soup.findAll('div', class_='spaceit_pad'):\n",
    "        if (div.span != None):\n",
    "            feature = div.getText(strip=True).split(':', maxsplit=1)\n",
    "            if (feature[0] == 'Theme'):\n",
    "                a_dict['Themes'] = feature[1]\n",
    "            else:\n",
    "                a_dict[feature[0]] = feature[1]\n",
    "\n",
    "    # # Fetch characters\n",
    "    # a_characters = []\n",
    "    # for h3 in a_soup.find_all('h3', class_='h3_characters_voice_actors'):\n",
    "    #     tmp = h3.getText(strip=True).split(', ')\n",
    "    #     a_characters.append(' '.join(tmp[::-1]))\n",
    "    # a_dict['Characters'] = ','.join(a_characters)\n",
    "\n",
    "    # # Fetch voice actors\n",
    "    # a_voice_actors = []\n",
    "    # for td in a_soup.find_all('td', class_='va-t ar pl4 pr4'):\n",
    "    #     tmp = td.a.getText(strip=True).split(', ')\n",
    "    #     a_voice_actors.append(' '.join(tmp[::-1]))\n",
    "    # a_dict['Voice Actors'] = ','.join(a_voice_actors)\n",
    "\n",
    "    # Filter dictionary\n",
    "    a_features = {}\n",
    "    for (key, value) in a_dict.items():\n",
    "            if key in req_features_list:\n",
    "                a_features[key] = value\n",
    "    a_dict = a_features\n",
    "\n",
    "    # Populate and return dataframe\n",
    "    a_col_names = pd.read_csv(file_path_1, sep=';').columns\n",
    "    return pd.DataFrame(columns=a_col_names).append(pd.Series(a_dict, name=0))\n",
    "\n",
    "\n",
    "def extract_reviews(a_soup):\n",
    "    # Init empty dict\n",
    "    r_dict = {}\n",
    "\n",
    "    # Fetch title & id\n",
    "    r_dict['anime_id'] = a_soup.find('div', id='content').find('a')['href'].split(sep='/')[4]\n",
    "    r_dict['Title'] = a_soup.find('h1', class_='title-name h1_bold_none').get_text()\n",
    "    \n",
    "    # Fetch top 5 reviews\n",
    "    r_count = 0\n",
    "    for div in a_soup.findAll('div' , attrs={'class':'spaceit textReadability word-break pt8 mt8'}):\n",
    "        text_0 = div.contents[2].strip()\n",
    "        text_1 = div.contents[3].get_text(strip=True)\n",
    "        r_dict[f'review_{r_count}'] = ''.join([text_0, ' ', text_1])\n",
    "        r_count += 1\n",
    "\n",
    "    # Populate dataframe and return\n",
    "    return pd.DataFrame(r_dict, index=[0])\n",
    "\n",
    "\n",
    "def scrap_url(url):\n",
    "    # Fetch page from URL\n",
    "    a_page = requests.get(url, headers=headers)\n",
    "    time.sleep(random.uniform(0, wait_time))\n",
    "\n",
    "    if (a_page.status_code != 200):\n",
    "        print('SERVER OVERLOAD', end='\\r')\n",
    "    \n",
    "    # Request agian after 120s if Error 403 is hit\n",
    "    while (a_page.status_code == 403):\n",
    "        time.sleep(120)\n",
    "        a_page = requests.get(url, headers=headers)\n",
    "\n",
    "    # Parse the request response\n",
    "    a_soup = BeautifulSoup(a_page.content, 'html.parser')\n",
    "    \n",
    "    # Remove hidden tags\n",
    "    for tag in a_soup.findAll('span', attrs={'style':'display: none'}):\n",
    "        tag.clear()\n",
    "\n",
    "    # Fetch anime-info features and write to `anime-info.csv`\n",
    "    a_features_df = extract_features(a_soup)\n",
    "    a_features_df.to_csv(file_path_1, mode='a', sep=';', index=False, header=False)\n",
    "\n",
    "    # Fetch anime reviews and wrtie to `anime-reviews.csv`\n",
    "    a_reviews_df = extract_reviews(a_soup)\n",
    "    a_reviews_df.to_csv(file_path_2, mode='a', sep=';', index=False, header=False)\n",
    "\n",
    "\n",
    "def build_dataset(anime_urls):\n",
    "    threads = min(MAX_THREADS, len(anime_urls))\n",
    "\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:\n",
    "        executor.map(scrap_url, anime_urls)\n",
    "\n",
    "def build_dataset_multiprocess(urls, n_cores):\n",
    "    processes = []\n",
    "    url_lists = np.array_split(urls, n_cores)\n",
    "\n",
    "    for i in range(n_cores):\n",
    "        processes.append(multiprocessing.Process(target=build_dataset, args=(url_lists[i].tolist(), )))\n",
    "        \n",
    "    for i in range(n_cores):\n",
    "        processes[i].start()\n",
    "        \n",
    "    for i in range(n_cores):\n",
    "        processes[i].join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "wait_time = 1\n",
    "\n",
    "# Define file paths\n",
    "file_path_1 = './Datasets/anime-info.csv'\n",
    "file_path_2 = './Datasets/anime-reviews.csv'\n",
    "url_file_path = './Datasets/anime-urls-main.csv'\n",
    "\n",
    "# Init anime-info dataset\n",
    "df = pd.DataFrame(\n",
    "    columns=['anime_id', 'Title', 'Synonyms', 'Type', 'Episodes', 'Status', 'Aired', 'Premiered', 'Broadcast', 'Producers', 'Licensors', 'Studios', 'Source', 'Genres', 'Themes', 'Demographic', 'Duration', 'Rating', 'Score', 'Ranked', 'Popularity', 'Members', 'Favorites'],\n",
    "    dtype=object\n",
    ")\n",
    "df.to_csv(file_path_1, sep=';', index=False, header=df.columns)\n",
    "\n",
    "# Init anime-review dataset\n",
    "df = pd.DataFrame(\n",
    "    columns=['anime_id', 'Title', 'review_1','review_2','review_3','review_4','review_5'],\n",
    "    dtype=object\n",
    ")\n",
    "df.to_csv(file_path_2, sep=';', index=False, header=df.columns)\n",
    "\n",
    "headers = {\n",
    "    'Host': 'myanimelist.net',\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',\n",
    "    'Accept-Language': 'en-US,en;q=0.5',\n",
    "    'Accept-Encoding': 'gzip, deflate, br',\n",
    "    'Referer': 'https://myanimelist.net/anime.php',\n",
    "    'Connection': 'keep-alive'\n",
    "}\n",
    "\n",
    "MAX_THREADS = 30\n",
    "\n",
    "sample_urls = [\n",
    "    'https://myanimelist.net/anime/5114/Fullmetal_Alchemist__Brotherhood',\n",
    "    'https://myanimelist.net/anime/38524/Shingeki_no_Kyojin_Season_3_Part_2',\n",
    "    'https://myanimelist.net/anime/37521/Vinland_Saga',\n",
    "    'https://myanimelist.net/anime/1535/Death_Note'\n",
    "]\n",
    "\n",
    "# Read URLs from CSV file written by `fetch_urls()`\n",
    "df_urls = pd.read_csv(url_file_path, header=None)\n",
    "urls = df_urls[0].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample run\n",
    "build_dataset_multiprocess(sample_urls * 8, n_cores=8)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}