Project_7_2_Youtube_Scrapping_Assignment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9713682c",
   "metadata": {},
   "source": [
    "## Go to this given URL and solve the following questions\n",
    "## URL: https://www.youtube.com/@PW-Foundation/videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ce2b260",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in /Users/rachit/anaconda3/lib/python3.10/site-packages (4.10.0)\n",
      "Requirement already satisfied: trio~=0.17 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from selenium) (0.22.1)\n",
      "Requirement already satisfied: certifi>=2021.10.8 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from selenium) (2022.12.7)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from selenium) (0.10.3)\n",
      "Requirement already satisfied: urllib3[socks]<3,>=1.26 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from selenium) (1.26.14)\n",
      "Requirement already satisfied: sortedcontainers in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: sniffio in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.0rc9 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (1.1.2)\n",
      "Requirement already satisfied: attrs>=20.1.0 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (22.1.0)\n",
      "Requirement already satisfied: outcome in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: idna in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio~=0.17->selenium) (3.4)\n",
      "Requirement already satisfied: wsproto>=0.14 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from trio-websocket~=0.9->selenium) (1.2.0)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.14.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c34f9fdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in /Users/rachit/anaconda3/lib/python3.10/site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from bs4) (4.11.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/rachit/anaconda3/lib/python3.10/site-packages (from beautifulsoup4->bs4) (2.3.2.post1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2e13f195",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import pandas as pd\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f82ad34e",
   "metadata": {},
   "outputs": [],
   "source": [
    "browser = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "348918b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.youtube.com/@PW-Foundation/videos\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e8c48a1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.get(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02a44d92",
   "metadata": {},
   "source": [
    "### Q1. Write a python program to extract the video URL of the first five videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "14d142cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.youtube.com/watch?v=dArUpCasmnE',\n",
       " 'https://www.youtube.com/watch?v=HqG2QchBw8Y',\n",
       " 'https://www.youtube.com/watch?v=1izKrQHyx9M',\n",
       " 'https://www.youtube.com/watch?v=jXAb1evxaJc',\n",
       " 'https://www.youtube.com/watch?v=2dn7XMxRtPE']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Finding Video Url by using Tag Name\n",
    "links = browser.find_elements(By.TAG_NAME,\"a\")\n",
    "video_urls = []\n",
    "for i in links:\n",
    "    url = i.get_attribute(\"href\")\n",
    "    video_urls.append(url)\n",
    "    \n",
    "first_5_video_links = []\n",
    "for url in video_urls:\n",
    "    if url == None or \"/watch?v=\" not in url or url in first_5_video_links:\n",
    "        continue\n",
    "    else:\n",
    "        if len(first_5_video_links) == 5:\n",
    "            break\n",
    "        first_5_video_links.append(url)\n",
    "first_5_video_links"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abce1d8d",
   "metadata": {},
   "source": [
    "### Q2. Write a python program to extract the URL of the video thumbnails of the first five videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8f93e71e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://i.ytimg.com/vi/dArUpCasmnE/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLCdiUURSFwzHKBaqzNQnNYVFf1PZA',\n",
       " 'https://i.ytimg.com/vi/HqG2QchBw8Y/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLB2qRg5GsfQROPt6YiiG3CXXiExjg',\n",
       " 'https://i.ytimg.com/vi/1izKrQHyx9M/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLCvjSUCwD4j5ZVE_nUMLg6QNCpkfg',\n",
       " 'https://i.ytimg.com/vi/jXAb1evxaJc/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLBs0CPJgSnMgMMu57VSlYuPtsxS5w',\n",
       " 'Url Not found']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thumbnail = browser.find_elements(By.XPATH,\"//*[@id='thumbnail']/yt-image/img\")\n",
    "thumbnail_list=[]\n",
    "for i in thumbnail:\n",
    "    t=i.get_attribute(\"src\")\n",
    "    if t == None:\n",
    "        continue\n",
    "    thumbnail_list.append(t)\n",
    "    \n",
    "thumbnail_list.append(\"Url Not found\")\n",
    "thumbnail_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fae7af69",
   "metadata": {},
   "source": [
    "### Q3. Write a python program to extract the title of the first five videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "498bd2bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Launching PAHAL Batch 🔥 For Class 11th Arts Students 💪',\n",
       " 'Launching FUNDO For Class - 6th to 8th Students 🔥💯 || Ab Hoga Padhai Ke Sath FUN 🤩',\n",
       " '⚡Unleashing the Power of PW Internationally⚡| Launching Physics Wallah Gulf',\n",
       " 'Big Announcement for Gulf Region Aspirants 🔥| Physics Wallah Gulf Channel Trailer🚀',\n",
       " 'Arjuna JEE v/s Arjuna NEET 🏏- Class 11th Faculties ka Cricket Match 🔥']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Finding Title by using TagName\n",
    "yt_link=browser.find_elements(By.TAG_NAME, \"yt-formatted-string\")\n",
    "link =[]\n",
    "for i in yt_link:\n",
    "    temp=i.text\n",
    "    link.append(temp)\n",
    "link=link[26:]\n",
    "title = []\n",
    "for i in link:\n",
    "    if i == '':\n",
    "        continue\n",
    "    else:\n",
    "        if len(title)==5:\n",
    "            break\n",
    "        title.append(i)\n",
    "        \n",
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "56354cd0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Launching PAHAL Batch 🔥 For Class 11th Arts Students 💪',\n",
       " 'Launching FUNDO For Class - 6th to 8th Students 🔥💯 || Ab Hoga Padhai Ke Sath FUN 🤩',\n",
       " '⚡Unleashing the Power of PW Internationally⚡| Launching Physics Wallah Gulf',\n",
       " 'Big Announcement for Gulf Region Aspirants 🔥| Physics Wallah Gulf Channel Trailer🚀',\n",
       " 'Arjuna JEE v/s Arjuna NEET 🏏- Class 11th Faculties ka Cricket Match 🔥']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Finding First 5 Title by using XPath\n",
    "you_link = browser.find_elements(By.XPATH,\"//*[@id='video-title']\")\n",
    "title_list = []\n",
    "for i in you_link:\n",
    "    t=i.text\n",
    "    if len(title_list) ==5:\n",
    "        break\n",
    "    title_list.append(t)\n",
    "    \n",
    "title_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be1b3f1f",
   "metadata": {},
   "source": [
    "### Q4. Write a python program to extract the number of views of the first five videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9d335165",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['21K views', '21K views', '65K views', '97K views', '375K views']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "views=browser.find_elements(By.XPATH,\"//*[@id='metadata-line']/span[1]\")\n",
    "views_list=[]\n",
    "for i in views:\n",
    "    t=i.text\n",
    "    if len(views_list) ==5:\n",
    "        break\n",
    "    views_list.append(t)\n",
    "    \n",
    "views_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "960f9d8b",
   "metadata": {},
   "source": [
    "### Q5. Write a python program to extract the time of posting of video for the first five videos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4eb88774",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['2 days ago', '6 days ago', '3 weeks ago', '1 month ago', '1 month ago']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "timee = browser.find_elements(By.XPATH,\"//*[@id='metadata-line']/span[2]\")\n",
    "time_list = []\n",
    "for i in timee:\n",
    "    t=i.text\n",
    "    if len(time_list) ==5:\n",
    "        break\n",
    "    time_list.append(t)\n",
    "    \n",
    "time_list"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a154866d",
   "metadata": {},
   "source": [
    "### Q6. Save all the data scraped in the above questions in a CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "44c85159",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cca73e2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews=[]\n",
    "for i in range(5):\n",
    "        mydict = {\"VideoLink\": str(first_5_video_links[i]),\"ThumbnailUrl\": str(thumbnail_list[i]),\"VideoTitle\": str(title_list[i]), \"Views\": str(views_list[i]), \"Time of Upload\": str(time_list[i])}\n",
    "        reviews.append(mydict)\n",
    "\n",
    "with open(\"Video_details.csv\", \"w\", newline='', encoding='utf-8') as fw:\n",
    "    headers = [\"VideoLink\",\"ThumbnailUrl\",\"VideoTitle\",\"Views\", \"Time of Upload\"]\n",
    "    writer = csv.DictWriter(fw, fieldnames=headers)\n",
    "    writer.writeheader()\n",
    "    writer.writerows(reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3b548100",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['VideoLink', 'ThumbnailUrl', 'VideoTitle', 'Views', 'Time of Upload']\n",
      "['https://www.youtube.com/watch?v=dArUpCasmnE', 'https://i.ytimg.com/vi/dArUpCasmnE/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLCdiUURSFwzHKBaqzNQnNYVFf1PZA', 'Launching PAHAL Batch 🔥 For Class 11th Arts Students 💪', '21K views', '2 days ago']\n",
      "['https://www.youtube.com/watch?v=HqG2QchBw8Y', 'https://i.ytimg.com/vi/HqG2QchBw8Y/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLB2qRg5GsfQROPt6YiiG3CXXiExjg', 'Launching FUNDO For Class - 6th to 8th Students 🔥💯 || Ab Hoga Padhai Ke Sath FUN 🤩', '21K views', '6 days ago']\n",
      "['https://www.youtube.com/watch?v=1izKrQHyx9M', 'https://i.ytimg.com/vi/1izKrQHyx9M/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLCvjSUCwD4j5ZVE_nUMLg6QNCpkfg', '⚡Unleashing the Power of PW Internationally⚡| Launching Physics Wallah Gulf', '65K views', '3 weeks ago']\n",
      "['https://www.youtube.com/watch?v=jXAb1evxaJc', 'https://i.ytimg.com/vi/jXAb1evxaJc/hqdefault.jpg?sqp=-oaymwEcCNACELwBSFXyq4qpAw4IARUAAIhCGAFwAcABBg==&rs=AOn4CLBs0CPJgSnMgMMu57VSlYuPtsxS5w', 'Big Announcement for Gulf Region Aspirants 🔥| Physics Wallah Gulf Channel Trailer🚀', '97K views', '1 month ago']\n",
      "['https://www.youtube.com/watch?v=2dn7XMxRtPE', 'Url Not found', 'Arjuna JEE v/s Arjuna NEET 🏏- Class 11th Faculties ka Cricket Match 🔥', '375K views', '1 month ago']\n"
     ]
    }
   ],
   "source": [
    "with open(\"Video_details.csv\",'r') as f:\n",
    "    read = csv.reader(f)\n",
    "    for i in read:\n",
    "        print(i)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}