03. Scraping_books.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4 as bs\n",
    "import urllib.request\n",
    "import csv\n",
    "\n",
    "page = 1\n",
    "alltitle = [['id', 'book_id', 'authors', 'title', 'average_rating', 'ratings_count']]\n",
    "\n",
    "while page != 9:\n",
    "\n",
    "    url = 'https://www.goodreads.com/list/show/567.Best_Chapter_Books_to_Read_Out_Loud?page=' + str(++page)\n",
    "\n",
    "    grURL = urllib.request.urlopen(url)\n",
    "    grHTML = grURL.read()\n",
    "    grURL.close()\n",
    "\n",
    "    soup = bs.BeautifulSoup(grHTML, 'html.parser')\n",
    "    #grAllSpans = soup.find_all(\"span\")\n",
    "    spans = soup.find_all('a', title= True)\n",
    "    for links in spans:\n",
    "        alltitle.append(links['id'], links['book_id'], links['authors'], links['title'], links['average_rating'], links['ratings_count'])\n",
    "                   \n",
    "        myFile = open('books.csv', 'w')\n",
    "        with myFile:\n",
    "            writer = csv.writer(alltitle)\n",
    "            writer.writerows(alltitle)\n",
    "            \n",
    "    if page == 9:\n",
    "        break\n",
    "    page += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>ratings_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2767052</td>\n",
       "      <td>Suzanne Collins</td>\n",
       "      <td>The Hunger Games (The Hunger Games, #1)</td>\n",
       "      <td>4.34</td>\n",
       "      <td>4780653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (Harry P...</td>\n",
       "      <td>4.44</td>\n",
       "      <td>4602479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>41865</td>\n",
       "      <td>Stephenie Meyer</td>\n",
       "      <td>Twilight (Twilight, #1)</td>\n",
       "      <td>3.57</td>\n",
       "      <td>3866839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2657</td>\n",
       "      <td>Harper Lee</td>\n",
       "      <td>To Kill a Mockingbird</td>\n",
       "      <td>4.25</td>\n",
       "      <td>3198671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4671</td>\n",
       "      <td>F. Scott Fitzgerald</td>\n",
       "      <td>The Great Gatsby</td>\n",
       "      <td>3.89</td>\n",
       "      <td>2683664</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  book_id                      authors  \\\n",
       "0   1  2767052              Suzanne Collins   \n",
       "1   2        3  J.K. Rowling, Mary GrandPré   \n",
       "2   3    41865              Stephenie Meyer   \n",
       "3   4     2657                   Harper Lee   \n",
       "4   5     4671          F. Scott Fitzgerald   \n",
       "\n",
       "                                               title  average_rating  \\\n",
       "0            The Hunger Games (The Hunger Games, #1)            4.34   \n",
       "1  Harry Potter and the Sorcerer's Stone (Harry P...            4.44   \n",
       "2                            Twilight (Twilight, #1)            3.57   \n",
       "3                              To Kill a Mockingbird            4.25   \n",
       "4                                   The Great Gatsby            3.89   \n",
       "\n",
       "   ratings_count  \n",
       "0        4780653  \n",
       "1        4602479  \n",
       "2        3866839  \n",
       "3        3198671  \n",
       "4        2683664  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books_data = pd.read_csv('books.csv')\n",
    "\n",
    "books_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}