Web Scraping-BS4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task: Extract all relevant information about the movies from the webpage:https://www.imdb.com/search/title/?count=100&groups=top_1000&sort=user_rating%27"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://help.imdb.com/article/contribution/titles/certificates/GU757M8ZJ9ZPXB39?ref_=helpart_nav_27#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd #pip install pandas\n",
    "from datetime import date\n",
    "from datetime import datetime\n",
    "from bs4 import BeautifulSoup #pip install BeautifulSoup4\n",
    "from tqdm import tqdm_notebook as tqdm #pip install tqdm\n",
    "from urllib.request import urlopen, Request\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "webpage='https://www.imdb.com/search/title/?count=100&groups=top_1000&sort=user_rating%27'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',\n",
    "       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n",
    "       'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',\n",
    "       'Accept-Encoding': 'none',\n",
    "       'Accept-Language': 'en-US,en;q=0.8',\n",
    "       'Connection': 'keep-alive'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_soup_object(link):\n",
    "    request=Request(link, headers=hdr)\n",
    "    html = urlopen(request).read().decode()\n",
    "    soup = BeautifulSoup(html,'html.parser')\n",
    "    return soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_links(page):\n",
    "    soup=extract_soup_object(page)\n",
    "    data=soup.find_all('h3',{'class':\"lister-item-header\"})\n",
    "    return ['https://www.imdb.com'+item.a.get('href') for item in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_links=extract_links(webpage)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Making Functions of all the attributes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Extract Scraping Time\n",
    "def get_scraping_date():\n",
    "    now = datetime.now()\n",
    "    date=now.strftime(\"%d-%m-%Y %H:%M:%S\")\n",
    "    return date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Movie name\n",
    "def movie_name(soup):\n",
    "    return soup.find('div',{'class':'title_wrapper'}).find('h1').text.split('\\xa0')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3-7. \n",
    "def other_attributes_part1(soup):\n",
    "    temp=soup.find('div',{'class':'title_wrapper'}).find('div',{'class':'subtext'}).text.replace(' ','').split('|')\n",
    "    certification=temp[0].split('\\n')[1]\n",
    "    time=temp[1].split('\\n')[1]\n",
    "    genre=temp[2].replace('\\n','')\n",
    "    release_date=temp[3].split('\\n')[1].split('(')[0]\n",
    "    release_country=temp[3].split('\\n')[1].split('(')[1].split(')')[0]\n",
    "    return certification, time, genre, release_date, release_country"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 8 Rating\n",
    "def movie_rating(soup):\n",
    "    return soup.find('div',{'class':'ratingValue'}).text.replace('\\n','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 9,10\n",
    "def user_critic(soup):\n",
    "    q=soup.find('div',{'class':'imdbRating'}).find('div',{'class':'hiddenImportant'}).find_all('span')\n",
    "    temp=[item.text for item in q]\n",
    "    user,critic=temp[0].split(' ')[0],temp[1].split(' ')[0]\n",
    "    return user, critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 11. Summary\n",
    "def summary(soup):\n",
    "    return soup.find('div',{'class':'plot_summary'}).find('div',{'class':'summary_text'}).text.replace('\\n','').replace('  ','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#12-14\n",
    "def writer_director_stars(soup):\n",
    "    t=soup.find_all('div',{'class':'credit_summary_item'})\n",
    "    temp=[item.text for item in t]\n",
    "    director=temp[0].split(':')[1].replace('\\n','')\n",
    "    writer=temp[1].split(':')[1].replace('\\n','')\n",
    "    primary_actor=temp[2].split(':')[1].replace('\\n','').split('|')[0]\n",
    "    return director, writer, primary_actor\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#15 meta score\n",
    "def meta_score(soup):\n",
    "    return soup.find('div',{'class':'metacriticScore score_favorable titleReviewBarSubItem'}).text.replace('\\n','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 16 primary image\n",
    "def primary_image(soup):\n",
    "    return soup.find('div',{'class':'poster'}).find('img')['src']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 17 primary video\n",
    "def primary_video(soup):\n",
    "    return 'https://www.imdb.com'+soup.find('div',{'class':'videoPreview__videoContainer'}).find('a')['href']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 18,19 other images, video\n",
    "def other_image_video(soup):\n",
    "    temp=soup.find_all('div',{'class':'combined-see-more see-more'})\n",
    "    temp_1=[item.find('a')['href'] for item in temp]\n",
    "    image=temp_1[0]\n",
    "    video=temp_1[1]\n",
    "    return image, video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 20 all actors\n",
    "def all_actors(soup, value):\n",
    "    temp=soup.find_all('tr', {'class':value})\n",
    "    return [item.text for item in temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beautify(temp_list):\n",
    "    actor=[]\n",
    "    character=[]\n",
    "    for item in temp_list:\n",
    "        actor.append(item.split('...')[0])\n",
    "        character.append(item.split('...')[1])\n",
    "    actors=[item.replace('\\n','').replace(' ','') for item in actor]\n",
    "    characters=[item.replace('\\n','').replace(' ','') for item in character]\n",
    "    return actors, characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#21. plot\n",
    "def plot(soup):\n",
    "    return soup.find('div',{'class':'inline canwrap'}).find('span').text.replace('  ','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 22. \n",
    "def plot_keywords(soup):\n",
    "    keywords=[]\n",
    "    temp=soup.find('div',{'class':'see-more inline canwrap'}).text.replace('\\n','')\n",
    "    keywords.append(temp.split('|')[0].split(':')[1])\n",
    "    keywords.extend(temp.split('|')[1:-1])\n",
    "    return keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 23-31\n",
    "def other_attributes_part2(soup):\n",
    "    h=soup.find('div',{'id':'titleDetails'}).find_all('div',{'class':'txt-block'})\n",
    "    details=[]\n",
    "    for item in h:\n",
    "        try:\n",
    "            details.append(item.text.split(':')[1])\n",
    "        except:\n",
    "            continue\n",
    "    languages=details[2].replace('\\n','').split('|')\n",
    "    filming_location=details[5].replace('\\n','').replace('\\xa0»','')\n",
    "    budget=details[6].replace('\\n','').split('(')[0]\n",
    "    opening_weekend=details[7].replace('\\n','')\n",
    "    gross_amount=details[8].replace('  ','')\n",
    "    cumilative_gross=details[9].replace('  ','')\n",
    "    production_company=details[10].replace('\\n','').replace('\\xa0»','')\n",
    "    sound_mix=details[12].replace('\\n','').split('|')\n",
    "    aspect_ratio=details[-1]\n",
    "    return languages, filming_location, budget, opening_weekend, gross_amount, cumilative_gross, production_company, sound_mix,aspect_ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imdb_scrapper(link):\n",
    "    data={}\n",
    "    data['website']='IMDB'\n",
    "    data['link']=link\n",
    "\n",
    "    scrape_date=get_scraping_date()\n",
    "    data['scrape_date']=scrape_date\n",
    "    \n",
    "    name=link.split('/')[4]\n",
    "    data['unique_id']=name\n",
    "    \n",
    "    soup=extract_soup_object(link)\n",
    "    \n",
    "    try:\n",
    "        data['movie_name']=movie_name(soup)\n",
    "    except:\n",
    "        data['movie_name']=None\n",
    "        \n",
    "    try:\n",
    "        certification, time, genre, release_date, release_country=other_attributes_part1(soup)\n",
    "        data['certification']=certification\n",
    "        data['time']=time\n",
    "        data['genre']=genre\n",
    "        data['release_date']=release_date\n",
    "        data['release_country']=release_country\n",
    "    except:\n",
    "        data['certification']=None\n",
    "        data['time']=None\n",
    "        data['genre']=None\n",
    "        data['release_date']=None\n",
    "        data['release_country']=None\n",
    "        \n",
    "    try:\n",
    "        data['movie_rating']=movie_rating(soup)\n",
    "    except:\n",
    "        data['movie_rating']=None\n",
    "        \n",
    "    try:\n",
    "        user, critic=user_critic(soup)\n",
    "        data['user']=user\n",
    "        data['critic']=critic\n",
    "    except:\n",
    "        data['user']=None\n",
    "        data['critic']=None\n",
    "\n",
    "        \n",
    "    try:\n",
    "        data['summary']=summary(soup)\n",
    "    except:\n",
    "        data['summary']=None\n",
    "        \n",
    "    try:\n",
    "        director, writer, primary_actor=writer_director_stars(soup)\n",
    "        data['director']=director\n",
    "        data['writer']=writer\n",
    "        data['primary_actor']=primary_actor\n",
    "    except:\n",
    "        data['director']=None\n",
    "        data['writer']=None\n",
    "        data['primary_actor']=None\n",
    "        \n",
    "    try:\n",
    "        data['meta_score']=meta_score(soup)\n",
    "    except:\n",
    "        data['meta_score']=None\n",
    "        \n",
    "    try:\n",
    "        data['primary_image']=primary_image(soup)\n",
    "    except:\n",
    "        data['primary_image']=None\n",
    "        \n",
    "    try:\n",
    "        data['primary_video']=primary_video(soup)\n",
    "    except:\n",
    "        data['primary_video']=None\n",
    "        \n",
    "    try:\n",
    "        image_links,video_links=other_image_video(soup)\n",
    "        data['other_image_links']=image_links\n",
    "        data['other_video_links']=video_links\n",
    "    except:\n",
    "        data['other_image_links']=None\n",
    "        data['other_video_links']=None\n",
    "        \n",
    "    try:\n",
    "        odd_actors=all_actors(soup, 'odd')\n",
    "        even_actors=all_actors(soup, 'even')\n",
    "        actors=odd_actors+even_actors\n",
    "        actors, characters=beautify(actors)\n",
    "        data['all_actors']=actors\n",
    "        data['all_characters']=characters\n",
    "    except:\n",
    "        data['all_actors']=None\n",
    "        data['all_characters']=None\n",
    "        \n",
    "    try:\n",
    "        data['plot']=plot(soup)\n",
    "    except:\n",
    "        data['plot']=None\n",
    "        \n",
    "    try:\n",
    "        data['plot_keywords']=plot_keywords(soup)\n",
    "    except:\n",
    "        data['plot_keywords']=None\n",
    "        \n",
    "    try:\n",
    "        languages, filming_location, budget, opening_weekend, gross_amount, cumilative_gross, production_company, sound_mix,aspect_ratio=other_attributes_part2(soup)\n",
    "        data['languages']=languages\n",
    "        data['filming_location']=filming_location\n",
    "        data['budget']=budget\n",
    "        data['opening_weekend']=opening_weekend\n",
    "        data['gross_amount']=gross_amount\n",
    "        data['cumilative_gross']=cumilative_gross\n",
    "        data['production_company']=production_company\n",
    "        data['sound_mix']=sound_mix\n",
    "        data['aspect_ratio']=aspect_ratio\n",
    "    except:\n",
    "        data['languages']=None\n",
    "        data['filming_location']=None\n",
    "        data['budget']=None\n",
    "        data['opening_weekend']=None\n",
    "        data['gross_amount']=None\n",
    "        data['cumilative_gross']=None\n",
    "        data['production_company']=None\n",
    "        data['sound_mix']=None\n",
    "        data['aspect_ratio']=None\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'website': 'IMDB',\n",
       " 'link': 'https://www.imdb.com/title/tt8946378/?ref_=adv_li_tt',\n",
       " 'scrape_date': '19-06-2020 20:20:40',\n",
       " 'unique_id': 'tt8946378',\n",
       " 'movie_name': 'Knives Out',\n",
       " 'certification': 'PG-13',\n",
       " 'time': '2h10min',\n",
       " 'genre': 'Comedy,Crime,Drama',\n",
       " 'release_date': '27November2019',\n",
       " 'release_country': 'USA',\n",
       " 'movie_rating': '7.9/10 ',\n",
       " 'user': '2,058',\n",
       " 'critic': '437',\n",
       " 'summary': 'A detective investigates the death of a patriarch of an eccentric, combative family.',\n",
       " 'director': 'Rian Johnson ',\n",
       " 'writer': 'Rian Johnson ',\n",
       " 'primary_actor': 'Daniel Craig, Chris Evans, Ana de Armas ',\n",
       " 'meta_score': '82',\n",
       " 'primary_image': 'https://m.media-amazon.com/images/M/MV5BMGUwZjliMTAtNzAxZi00MWNiLWE2NzgtZGUxMGQxZjhhNDRiXkEyXkFqcGdeQXVyNjU1NzU3MzE@._V1_UX182_CR0,0,182,268_AL__QL50.jpg',\n",
       " 'primary_video': None,\n",
       " 'other_image_links': '/title/tt8946378/videogallery?ref_=tt_pv_vi_sm',\n",
       " 'other_video_links': '/title/tt8946378/mediaindex?ref_=tt_pv_mi_sm',\n",
       " 'all_actors': ['DanielCraig',\n",
       "  'AnadeArmas',\n",
       "  'MichaelShannon',\n",
       "  'ToniCollette',\n",
       "  'ChristopherPlummer',\n",
       "  'JaedenMartell',\n",
       "  'EdiPatterson',\n",
       "  'KCallan',\n",
       "  'ChrisEvans',\n",
       "  'JamieLeeCurtis',\n",
       "  'DonJohnson',\n",
       "  'LaKeithStanfield',\n",
       "  'KatherineLangford',\n",
       "  'RikiLindhome',\n",
       "  'FrankOz'],\n",
       " 'all_characters': ['BenoitBlanc',\n",
       "  'MartaCabrera',\n",
       "  'WaltThrombey',\n",
       "  'JoniThrombey',\n",
       "  'HarlanThrombey',\n",
       "  'JacobThrombey',\n",
       "  'Fran',\n",
       "  'GreatnanaWanetta',\n",
       "  'RansomDrysdale',\n",
       "  'LindaDrysdale',\n",
       "  'RichardDrysdale',\n",
       "  'LieutenantElliott',\n",
       "  'MegThrombey',\n",
       "  'DonnaThrombey',\n",
       "  'AlanStevens'],\n",
       " 'plot': \"When renowned crime novelist Harlan Thrombey (Christopher Plummer) is found dead at his estate just after his 85th birthday, the inquisitive and debonair Detective Benoit Blanc (Daniel Craig) is mysteriously enlisted to investigate. From Harlan's dysfunctional family to his devoted staff, Blanc sifts through a web of red herrings and self-serving lies to uncover the truth behind Harlan's untimely death.\",\n",
       " 'plot_keywords': [' vomiting',\n",
       "  ' whodunit',\n",
       "  ' private detective',\n",
       "  ' police',\n",
       "  ' nurse'],\n",
       " 'languages': ['English', 'Spanish', 'Hindi'],\n",
       " 'filming_location': 'Boston, Massachusetts, USASee more      ',\n",
       " 'budget': '$40,000,000            ',\n",
       " 'opening_weekend': ' $26,769,548,1 December 2019 ',\n",
       " 'gross_amount': ' $165,363,234',\n",
       " 'cumilative_gross': ' $309,232,797',\n",
       " 'production_company': ' Lionsgate, Media Rights Capital (MRC), T-Street See more      ',\n",
       " 'sound_mix': ['Dolby', ' D-Cinema 48kHz 5.1', ' Dolby Atmos'],\n",
       " 'aspect_ratio': ' 1.85 '}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_scrapper('https://www.imdb.com/title/tt8946378/?ref_=adv_li_tt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in tqdm(movie_links):\n",
    "        data=imdb_scrapper(item)\n",
    "        name=item.split('/')[4]\n",
    "        with open(f'/home/shivangi/Desktop/movie_database/{name}.json', 'w',encoding='utf-8') as outfile:\n",
    "            json.dump(data, outfile,indent=4, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}