0307_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting seleniumNote: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip available: 22.3.1 -> 23.0.1\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  Downloading selenium-4.8.2-py3-none-any.whl (6.9 MB)\n",
      "     ---------------------------------------- 6.9/6.9 MB 4.8 MB/s eta 0:00:00\n",
      "Requirement already satisfied: urllib3[socks]~=1.26 in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from selenium) (1.26.2)\n",
      "Collecting trio-websocket~=0.9\n",
      "  Downloading trio_websocket-0.9.2-py3-none-any.whl (16 kB)\n",
      "Collecting trio~=0.17\n",
      "  Downloading trio-0.22.0-py3-none-any.whl (384 kB)\n",
      "     -------------------------------------- 384.9/384.9 kB 2.7 MB/s eta 0:00:00\n",
      "Collecting certifi>=2021.10.8\n",
      "  Downloading certifi-2022.12.7-py3-none-any.whl (155 kB)\n",
      "     -------------------------------------- 155.3/155.3 kB 1.8 MB/s eta 0:00:00\n",
      "Requirement already satisfied: attrs>=19.2.0 in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from trio~=0.17->selenium) (20.3.0)\n",
      "Collecting exceptiongroup>=1.0.0rc9\n",
      "  Downloading exceptiongroup-1.1.0-py3-none-any.whl (14 kB)\n",
      "Collecting outcome\n",
      "  Downloading outcome-1.2.0-py2.py3-none-any.whl (9.7 kB)\n",
      "Requirement already satisfied: idna in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from trio~=0.17->selenium) (2.10)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from trio~=0.17->selenium) (2.3.0)\n",
      "Collecting sniffio\n",
      "  Downloading sniffio-1.3.0-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from trio~=0.17->selenium) (1.14.4)\n",
      "Requirement already satisfied: async-generator>=1.9 in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from trio~=0.17->selenium) (1.10)\n",
      "Collecting wsproto>=0.14\n",
      "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from urllib3[socks]~=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: pycparser in c:\\users\\mnju5\\.conda\\envs\\testai\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.20)\n",
      "Collecting h11<1,>=0.9.0\n",
      "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
      "     ---------------------------------------- 58.3/58.3 kB 3.2 MB/s eta 0:00:00\n",
      "Installing collected packages: sniffio, outcome, h11, exceptiongroup, certifi, wsproto, trio, trio-websocket, selenium\n",
      "  Attempting uninstall: certifi\n",
      "    Found existing installation: certifi 2020.12.5\n",
      "    Uninstalling certifi-2020.12.5:\n",
      "      Successfully uninstalled certifi-2020.12.5\n",
      "Successfully installed certifi-2022.12.7 exceptiongroup-1.1.0 h11-0.14.0 outcome-1.2.0 selenium-4.8.2 sniffio-1.3.0 trio-0.22.0 trio-websocket-0.9.2 wsproto-1.2.0\n"
     ]
    }
   ],
   "source": [
    "pip install selenium"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 크롤링 \n",
    "- 조효진, 쯔양, 김아영, 주현영, 박세미(서준엄마), 풍자, 박막례할머니, 심으뜸, 가요이키우기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import urllib.request\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_term = '조효진'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"http://www.google.co.in/search?q=\" + search_term + \"&tbm=isch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-45db571071ce>:1: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  browser = webdriver.Chrome(\"chromedriver.exe\")\n"
     ]
    }
   ],
   "source": [
    "browser = webdriver.Chrome(\"chromedriver.exe\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(200):\n",
    "    browser.execute_script('window.scrollBy(0,10000)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "photo_list = [] #다운 받을 사진의 list 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(search_term):\n",
    "    os.mkdir(search_term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, img in enumerate(photo_list[0:]):\n",
    "    #태그의 리스트를 하나씩 클릭한다.\n",
    "    img.click()\n",
    "    \n",
    "    #확대된 이미지의 정보는 img태그의 _image_source라는 class안에 담겨있다.\n",
    "    html_objects = browser.find_element_by_tag_name('img._image_source')\n",
    "    current_src = html_objects.get_attribute('src')\n",
    "    \n",
    "    t = urlopen(current_src).read()\n",
    "    if index < 400 : #다운 받을 장 수 정하기\n",
    "        filename = searchterm+str(count)+\".jpg\"\n",
    "        File = open(os.path.join(searchterm , searchterm + \"_\" + str(count) + \".jpg\"), \"wb\")\n",
    "        File.write(t)\n",
    "        count += 1\n",
    "        print(\"img save\"+str(count))\n",
    "    else:\n",
    "        print(\"end\")\n",
    "        browser.close()       \n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "InvalidSessionIdException",
     "evalue": "Message: invalid session id\nStacktrace:\nBacktrace:\n\t(No symbol) [0x002537D3]\n\t(No symbol) [0x001E8B81]\n\t(No symbol) [0x000EB230]\n\t(No symbol) [0x00110758]\n\t(No symbol) [0x0013B316]\n\t(No symbol) [0x00138092]\n\t(No symbol) [0x00137544]\n\t(No symbol) [0x000C5427]\n\t(No symbol) [0x000C5B0E]\n\t(No symbol) [0x000C5EEA]\n\tGetHandleVerifier [0x004CABF2+2510930]\n\tGetHandleVerifier [0x004F8EC1+2700065]\n\tGetHandleVerifier [0x004FC86C+2714828]\n\tGetHandleVerifier [0x00303480+645344]\n\t(No symbol) [0x001F0FD2]\n\t(No symbol) [0x000C516A]\n\t(No symbol) [0x000C4B96]\n\tGetHandleVerifier [0x00518C9C+2830588]\n\tBaseThreadInitThunk [0x75C47D69+25]\n\tRtlInitializeExceptionChain [0x7700BB9B+107]\n\tRtlClearBits [0x7700BB1F+191]\n\t(No symbol) [0x00000000]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvalidSessionIdException\u001b[0m                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-e85cfaaef3b1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbrowser\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_elements\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCLASS_NAME\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"rg_ic\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m     \u001b[0mel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscreenshot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\"png\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\testAI\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_elements\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    858\u001b[0m         \u001b[1;31m# Return empty list if driver returns null\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    859\u001b[0m         \u001b[1;31m# See https://github.com/SeleniumHQ/selenium/issues/4555\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 860\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFIND_ELEMENTS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"using\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"value\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    861\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    862\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\testAI\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    438\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 440\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    441\u001b[0m             \u001b[0mresponse\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unwrap_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    442\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\testAI\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    243\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"alert\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"text\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 245\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mInvalidSessionIdException\u001b[0m: Message: invalid session id\nStacktrace:\nBacktrace:\n\t(No symbol) [0x002537D3]\n\t(No symbol) [0x001E8B81]\n\t(No symbol) [0x000EB230]\n\t(No symbol) [0x00110758]\n\t(No symbol) [0x0013B316]\n\t(No symbol) [0x00138092]\n\t(No symbol) [0x00137544]\n\t(No symbol) [0x000C5427]\n\t(No symbol) [0x000C5B0E]\n\t(No symbol) [0x000C5EEA]\n\tGetHandleVerifier [0x004CABF2+2510930]\n\tGetHandleVerifier [0x004F8EC1+2700065]\n\tGetHandleVerifier [0x004FC86C+2714828]\n\tGetHandleVerifier [0x00303480+645344]\n\t(No symbol) [0x001F0FD2]\n\t(No symbol) [0x000C516A]\n\t(No symbol) [0x000C4B96]\n\tGetHandleVerifier [0x00518C9C+2830588]\n\tBaseThreadInitThunk [0x75C47D69+25]\n\tRtlInitializeExceptionChain [0x7700BB9B+107]\n\tRtlClearBits [0x7700BB1F+191]\n\t(No symbol) [0x00000000]\n"
     ]
    }
   ],
   "source": [
    "for idx, el in enumerate(browser.find_elements(By.CLASS_NAME, \"rg_ic\")):\n",
    "    el.screenshot(str(idx) + \"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'WebDriver' object has no attribute 'find_element_by_css_selector'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-424105167a1f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"input.gLFyf\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_keys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkeyword\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m#send keyword\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_css_selector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"input.gLFyf\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_keys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mRETURN\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'WebDriver' object has no attribute 'find_element_by_css_selector'"
     ]
    }
   ],
   "source": [
    "driver.find_element_by_css_selector(\"input.gLFyf\").send_keys(keyword) #send keyword\n",
    "driver.find_element_by_css_selector(\"input.gLFyf\").send_keys(Keys.RETURN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector(\"input.gLFyf\").send_keys(keyword"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}