PolovniAutomobili WebScraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import pandas as pd\n",
    "import  sys\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-2-7894f6f16619>:1: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome('C:/Users/Pc4y/Downloads/chromedriver.exe')\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome('C:/Users/Pc4y/Downloads/chromedriver.exe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that parser through page and returns bs content\n",
    "def parse_page(url):\n",
    "    try:\n",
    "        driver.get(url)\n",
    "        \n",
    "        content = driver.page_source\n",
    "        soup = bs(content,'html.parser')\n",
    "        return soup\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a list of all models of one brand\n",
    "def get_models(brand):\n",
    "\tbrand = (brand.replace(' ', '-')).lower()\n",
    "\turl ='https://www.polovniautomobili.com/auto-oglasi/pretraga?brand=' + brand\n",
    "\tsoup = parse_page(url)\n",
    "\tmodels= soup.find(id='model')\n",
    "\tmodels_list=[]\n",
    "\tfor model in models:\n",
    "\t\tif model.get_text()=='Ostalo':\n",
    "\t\t\tpass\n",
    "\t\telse:\n",
    "\t\t\tmodels_list.append(model.get_text())\n",
    "\treturn models_list[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace(string):\n",
    "\tstring = string.replace('š','s')\n",
    "\tstring = string.replace('Š','s')\n",
    "\tstring = string.replace('ć','c')\n",
    "\tstring = string.replace('Ć','c')\n",
    "\tstring = string.replace('Ž','z')\n",
    "\tstring = string.replace('ž','z')\n",
    "\tstring = string.replace('č','c')\n",
    "\tstring = string.replace('Č','c')\n",
    "\treturn string\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_num_of_cars(url):\n",
    "    soup = parse_page(url)\n",
    "    \n",
    "    if soup!=None:\n",
    "        try:\n",
    "            num = soup.find_all('small')\n",
    "            num = num[-2].get_text()\n",
    "            num = num.split(' ')[-1]\n",
    "            \n",
    "            return (round(int(num)/25)+1)\n",
    "        except:\n",
    "            return -1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all cars from one brand (iterate through each model in given brand)\n",
    "def get_cars(): #model\n",
    "\n",
    "    #brand = (brand.replace(' ', '-')).lower()\n",
    "    #model = (model.replace(' ', '-')).lower()\n",
    "    url1 ='https://www.polovniautomobili.com/auto-oglasi/pretraga?page='\n",
    "    url2 = '&sort=basic&price_from=1000&price_to=20000&year_from=2000&year_to=2020&chassis%5B0%5D=277&chassis%5B1%5D=2631&chassis%5B2%5D=278&chassis%5B3%5D=2636&chassis%5B4%5D=2632&city_distance=0&showOldNew=all&without_price=1&engine_volume_to=3000&mileage_from=50000&wheel_side=2630'\n",
    "    list_of_links = []\n",
    "\n",
    "    # find number of pages in a given model\n",
    "    num = get_num_of_cars(url1+str(1)+url2)\n",
    "    print(num)\n",
    "    \n",
    "    if num==0: \n",
    "        num=1\n",
    "    # if ther are car ads on site and number of cars is greater or equal to 100\n",
    "    if num!=-1:\n",
    "        carList = []\n",
    "        # loop over number of pages\n",
    "        for i in range(1,num+1):\n",
    "            #print('On page {}'.format(str(i)))\n",
    "            if i%50==0:\n",
    "                time.sleep(25)\n",
    "\n",
    "            soup = parse_page(url1+str(i)+url2)\n",
    "            \n",
    "            if soup!=None:\n",
    "                #print('Found articles')\n",
    "                try:\n",
    "                    for a in soup.find_all(class_=\"ga-title\", href=True):\n",
    "                        list_of_links.append(a['href'])\n",
    "                        \n",
    "\n",
    "                except:\n",
    "                    print('Change here')\n",
    "                    pass\n",
    "            \n",
    "                   \n",
    "        return list_of_links         \n",
    "    else:\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function that inserts cars into dataframe and then loads into .csv file\n",
    "def insertAll():\n",
    "    \n",
    "    # iterate through every brand and every model of that brand and load data into dataframe\n",
    "    links = []\n",
    "    duplicate_models = ['PASSAT','GOLF','CORSA','ASTRA', 'VECTRA', 'SERIJA X', 'SERIJA M','SERIJA 1',\n",
    "                        'SERIJA 2','SERIJA 3','SERIJA 4','SERIJA 5','SERIJA 7','SERIJA 6','SERIJA 8','SERIJA I','QUATTRO', 'R-X8']\n",
    "    try:\n",
    "        cars = get_cars()\n",
    "        if cars!=None:\n",
    "            links.append(cars)\n",
    "                    \n",
    "                               \n",
    "        return links\n",
    "    \n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        textfile = open(\"all.txt\", \"w\")\n",
    "        for element in links:\n",
    "            if element!=None:\n",
    "                for url in element:\n",
    "                    textfile.write(url + \"\\n\")\n",
    "        textfile.close()\n",
    "        pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#carList=[]\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "def scrape_pages(urls):\n",
    "    data = pd.DataFrame()\n",
    "    carList=[]\n",
    "    driver = webdriver.Chrome('C:/Users/Pc4y/Downloads/chromedriver.exe')\n",
    "    driver.set_page_load_timeout(8)\n",
    "    #print('Driver')\n",
    "    count = 1\n",
    "    \n",
    "    for url in urls:\n",
    "        if url!=None:\n",
    "\n",
    "            try:\n",
    "                if count%50==0:\n",
    "                    time.sleep(15)\n",
    "                driver.get('https://www.polovniautomobili.com'+url)\n",
    "                WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, \"infoBox\")))\n",
    "                content = driver.page_source\n",
    "                soup = bs(content,'html.parser')\n",
    "                #print(soup)\n",
    "                \n",
    "\n",
    "                try:\n",
    "                    cena = soup.find(class_=\"priceClassified regularPriceColor\")\n",
    "                    cena_popust = soup.find(\"priceClassified discountedPriceColor\")\n",
    "                    if cena.get_text()!=None:\n",
    "                        cena=cena.get_text()[:-2]\n",
    "                        cena = int(cena.split('.')[0]+cena.split('.')[1])\n",
    "                        cena_dict = {'Cena': cena}\n",
    "                    elif cena_popust.get_text()!=None:\n",
    "                        cena=cena_popust.get_text()\n",
    "                        cena = int(cena.split('.')[0]+cena.split('.')[1])\n",
    "                        cena_dict = {'Cena': cena}\n",
    "\n",
    "                    opste = soup.find_all(class_=\"infoBox\")\n",
    "                    labels = []\n",
    "                    info = []\n",
    "                    i = 1\n",
    "                    for kar in opste:\n",
    "                        naziv = kar.find_all(class_=\"uk-width-1-2\")\n",
    "                        #print('*'*20)\n",
    "\n",
    "                        if i == 1:\n",
    "                            labels = [label.get_text() for label in naziv[::2]]\n",
    "                            info = [info_.get_text() for info_ in naziv[1::2]]\n",
    "                            info_dict = dict(zip(labels, info))\n",
    "                            labels = []\n",
    "                            info = []\n",
    "                            i +=1\n",
    "                            #break\n",
    "                            \n",
    "                        \n",
    "                        elif i==2:\n",
    "                            labels = [label.get_text() for label in naziv[::2]]\n",
    "                            info = [info_.get_text() for info_ in naziv[1::2]]\n",
    "                            oprema_dict = dict(zip(labels, info))\n",
    "                            #print(oprema_dict)\n",
    "                            labels = []\n",
    "                            info = []\n",
    "                            break\n",
    "                            \n",
    "                        \n",
    "                    dict_car = {**info_dict,**cena_dict, **oprema_dict} # **oprema_dict, **dodatno, \n",
    "                    carList.append(dict_car)\n",
    "                    \n",
    "                except:\n",
    "                    #print(\"Unexpected error:\", sys.exc_info())\n",
    "                    #data = data.append(carList,ignore_index=True)\n",
    "                    #print(count)\n",
    "                    pass\n",
    "            except TimeoutException:\n",
    "                #print(\"Unexpected error, driver refreshed:\", sys.exc_info())\n",
    "                #driver.refresh()\n",
    "                #print(count)\n",
    "                pass\n",
    "        count+=1\n",
    "        \n",
    "    data = data.append(carList,ignore_index=True)\n",
    "    driver.close()\n",
    "    return data\n",
    "                    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'textfile = open(\"all.txt\", \"w\")\\nfor element in list_of_links:\\n    if element!=None:\\n        for url in element:\\n            textfile.write(url + \"\\n\")\\ntextfile.close()'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "#driver = webdriver.Chrome('C:/Users/Pc4y/Downloads/chromedriver.exe')\n",
    "import time \n",
    "\n",
    "#list_of_links = insertAll()\n",
    "\"\"\"textfile = open(\"all.txt\", \"w\")\n",
    "for element in list_of_links:\n",
    "    if element!=None:\n",
    "        for url in element:\n",
    "            textfile.write(url + \"\\n\")\n",
    "textfile.close()\"\"\"\n",
    "#print(len(lista))https://www.polovniautomobili.com/auto-oglasi/pretraga?brand=alfa-romeo&brand2=&price_from=1000&price_to=20000&year_from=2000&year_to=2020&chassis%5B%5D=277&chassis%5B%5D=2631&chassis%5B%5D=278&chassis%5B%5D=2632&flywheel=&atest=&door_num=&submit_1=&without_price=1&date_limit=&showOldNew=old&modeltxt=&engine_volume_from=&engine_volume_to=&power_from=&power_to=&mileage_from=&mileage_to=&emission_class=&seat_num=&wheel_side=&registration=&country=&country_origin=&city=&registration_price=&sort=&page=8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'list_of_links' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-34dd3a4cef23>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlist_of_links\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'list_of_links' is not defined"
     ]
    }
   ],
   "source": [
    "list_of_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('all.txt', 'r') as f:\n",
    "    lista = f.readlines()\n",
    "\n",
    "#lista_deo = lista[:3000]\n",
    "#lista_deo2 = lista[3000:6000]\n",
    "#lista_deo3 = lista[6000:7000]\n",
    "#lista_deo5 = lista[12000:15870]\n",
    "#print(len(lista))\n",
    "\n",
    "#data = scrape_pages(list_of_links[0][:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_urls = [k.split('\\n')[0] for k in lista]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39769"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18000 20000\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-71-83eddb1965b0>:10: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome('C:/Users/Pc4y/Downloads/chromedriver.exe')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "47\n",
      "57\n",
      "72\n",
      "83\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "109\n",
      "124\n",
      "140\n",
      "151\n",
      "179\n",
      "181\n",
      "194\n",
      "195\n",
      "214\n",
      "216\n",
      "228\n",
      "229\n",
      "236\n",
      "242\n",
      "256\n",
      "262\n",
      "269\n",
      "308\n",
      "367\n",
      "376\n",
      "404\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "462\n",
      "488\n",
      "514\n",
      "528\n",
      "542\n",
      "543\n",
      "544\n",
      "545\n",
      "550\n",
      "593\n",
      "646\n",
      "692\n",
      "693\n",
      "698\n",
      "718\n",
      "747\n",
      "757\n",
      "761\n",
      "766\n",
      "787\n",
      "812\n",
      "825\n",
      "826\n",
      "830\n",
      "836\n",
      "854\n",
      "866\n",
      "882\n",
      "884\n",
      "897\n",
      "919\n",
      "938\n",
      "952\n",
      "954\n",
      "966\n",
      "989\n",
      "996\n",
      "1040\n",
      "1046\n",
      "1094\n",
      "1113\n",
      "1126\n",
      "1141\n",
      "1142\n",
      "1189\n",
      "1211\n",
      "1218\n",
      "1239\n",
      "1267\n",
      "1286\n",
      "1287\n",
      "1288\n",
      "1289\n",
      "1290\n",
      "1291\n",
      "1292\n",
      "1313\n",
      "1315\n",
      "1329\n",
      "1340\n",
      "1358\n",
      "1382\n",
      "1384\n",
      "1405\n",
      "1413\n",
      "1424\n",
      "1438\n",
      "1459\n",
      "1525\n",
      "1578\n",
      "1622\n",
      "1627\n",
      "1629\n",
      "1630\n",
      "1633\n",
      "1637\n",
      "1638\n",
      "1679\n",
      "1707\n",
      "1727\n",
      "1745\n",
      "1783\n",
      "1792\n",
      "1798\n",
      "1799\n",
      "1827\n",
      "1830\n",
      "1863\n",
      "1996\n",
      "2000\n",
      "20000 22000\n",
      "11\n",
      "24\n",
      "39\n",
      "53\n",
      "72\n",
      "82\n",
      "97\n",
      "99\n",
      "110\n",
      "189\n",
      "233\n",
      "259\n",
      "270\n",
      "276\n",
      "306\n",
      "332\n",
      "355\n",
      "370\n",
      "399\n",
      "406\n",
      "409\n",
      "416\n",
      "425\n",
      "438\n",
      "443\n",
      "457\n",
      "458\n",
      "465\n",
      "468\n",
      "469\n",
      "491\n",
      "493\n",
      "501\n",
      "505\n",
      "528\n",
      "545\n",
      "546\n",
      "556\n",
      "557\n",
      "559\n",
      "567\n",
      "568\n",
      "585\n",
      "597\n",
      "609\n",
      "610\n",
      "612\n",
      "614\n",
      "615\n",
      "620\n",
      "630\n",
      "667\n",
      "672\n",
      "674\n",
      "681\n",
      "688\n",
      "693\n",
      "700\n",
      "704\n",
      "705\n",
      "760\n",
      "761\n",
      "763\n",
      "781\n",
      "782\n",
      "783\n",
      "805\n",
      "810\n",
      "873\n",
      "921\n",
      "957\n",
      "991\n",
      "997\n",
      "1018\n",
      "1045\n",
      "1047\n",
      "1048\n",
      "1052\n",
      "1059\n",
      "1060\n",
      "1068\n",
      "1079\n",
      "1105\n",
      "1111\n",
      "1130\n",
      "1138\n",
      "1143\n",
      "1181\n",
      "1191\n",
      "1195\n",
      "1252\n",
      "1266\n",
      "1274\n",
      "1279\n",
      "1303\n",
      "1309\n",
      "1312\n",
      "1321\n",
      "1327\n",
      "1341\n",
      "1369\n",
      "1371\n",
      "1374\n",
      "1378\n",
      "1379\n",
      "1380\n",
      "1386\n",
      "1390\n",
      "1394\n",
      "1395\n",
      "1405\n",
      "1420\n",
      "1437\n",
      "1510\n",
      "1524\n",
      "1536\n",
      "1543\n",
      "1548\n",
      "1558\n",
      "1563\n",
      "1570\n",
      "1601\n",
      "1641\n",
      "1649\n",
      "1652\n",
      "1659\n",
      "1676\n",
      "1704\n",
      "1736\n",
      "1740\n",
      "1761\n",
      "1769\n",
      "1772\n",
      "1786\n",
      "1798\n",
      "1799\n",
      "1813\n",
      "1831\n",
      "1832\n",
      "1839\n",
      "1860\n",
      "1864\n",
      "1870\n",
      "1873\n",
      "1883\n",
      "1884\n",
      "1934\n",
      "1938\n",
      "1975\n",
      "1981\n",
      "1983\n",
      "1993\n",
      "2000\n",
      "22000 24000\n",
      "12\n",
      "1\n",
      "9\n",
      "10\n",
      "11\n",
      "32\n",
      "38\n",
      "45\n",
      "56\n",
      "73\n",
      "79\n",
      "89\n",
      "117\n",
      "122\n",
      "128\n",
      "146\n",
      "185\n",
      "189\n",
      "190\n",
      "201\n",
      "221\n",
      "258\n",
      "264\n",
      "297\n",
      "307\n",
      "310\n",
      "318\n",
      "321\n",
      "344\n",
      "347\n",
      "376\n",
      "379\n",
      "386\n",
      "406\n",
      "407\n",
      "413\n",
      "417\n",
      "421\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "436\n",
      "446\n",
      "449\n",
      "450\n",
      "453\n",
      "454\n",
      "466\n",
      "473\n",
      "474\n",
      "491\n",
      "504\n",
      "506\n",
      "507\n",
      "508\n",
      "509\n",
      "510\n",
      "511\n",
      "518\n",
      "526\n",
      "541\n",
      "565\n",
      "581\n",
      "596\n",
      "604\n",
      "605\n",
      "606\n",
      "607\n",
      "610\n",
      "638\n",
      "648\n",
      "655\n",
      "666\n",
      "681\n",
      "683\n",
      "697\n",
      "705\n",
      "725\n",
      "726\n",
      "740\n",
      "759\n",
      "868\n",
      "875\n",
      "887\n",
      "894\n",
      "907\n",
      "909\n",
      "911\n",
      "912\n",
      "913\n",
      "914\n",
      "924\n",
      "927\n",
      "940\n",
      "959\n",
      "961\n",
      "967\n",
      "979\n",
      "981\n",
      "990\n",
      "993\n",
      "997\n",
      "999\n",
      "1003\n",
      "1015\n",
      "1030\n",
      "1031\n",
      "1037\n",
      "1041\n",
      "1063\n",
      "1070\n",
      "1094\n",
      "1099\n",
      "1121\n",
      "1122\n",
      "1125\n",
      "1127\n",
      "1128\n",
      "1129\n",
      "1130\n",
      "1131\n",
      "1132\n",
      "1133\n",
      "1134\n",
      "1135\n",
      "1136\n",
      "1137\n",
      "1138\n",
      "1139\n",
      "1140\n",
      "1162\n",
      "1170\n",
      "1200\n",
      "1203\n",
      "1205\n",
      "1206\n",
      "1207\n",
      "1208\n",
      "1209\n",
      "1210\n",
      "1211\n",
      "1212\n",
      "1213\n",
      "1214\n",
      "1215\n",
      "1216\n",
      "1217\n",
      "1218\n",
      "1219\n",
      "1220\n",
      "1222\n",
      "1240\n",
      "1249\n",
      "1267\n",
      "1278\n",
      "1289\n",
      "1304\n",
      "1308\n",
      "1314\n",
      "1333\n",
      "1345\n",
      "1362\n",
      "1366\n",
      "1385\n",
      "1414\n",
      "1453\n",
      "1460\n",
      "1462\n",
      "1497\n",
      "1498\n",
      "1499\n",
      "1545\n",
      "1568\n",
      "1576\n",
      "1585\n",
      "1610\n",
      "1623\n",
      "1647\n",
      "1648\n",
      "1677\n",
      "1689\n",
      "1698\n",
      "1699\n",
      "1700\n",
      "1701\n",
      "1702\n",
      "1716\n",
      "1744\n",
      "1766\n",
      "1768\n",
      "1785\n",
      "1786\n",
      "1787\n",
      "1796\n",
      "1816\n",
      "1817\n",
      "1828\n",
      "1844\n",
      "1846\n",
      "1848\n",
      "1854\n",
      "1876\n",
      "1890\n",
      "1910\n",
      "1911\n",
      "1912\n",
      "1913\n",
      "1914\n",
      "1915\n",
      "1916\n",
      "1919\n",
      "1943\n",
      "1949\n",
      "1951\n",
      "24000 26000\n",
      "13\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "9\n",
      "32\n",
      "43\n",
      "45\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "94\n",
      "95\n",
      "109\n",
      "110\n",
      "113\n",
      "114\n",
      "115\n",
      "117\n",
      "132\n",
      "134\n",
      "137\n",
      "150\n",
      "178\n",
      "200\n",
      "204\n",
      "217\n",
      "223\n",
      "227\n",
      "240\n",
      "260\n",
      "273\n",
      "274\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "334\n",
      "352\n",
      "355\n",
      "361\n",
      "378\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "392\n",
      "393\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "410\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "427\n",
      "436\n",
      "472\n",
      "476\n",
      "492\n",
      "500\n",
      "526\n",
      "548\n",
      "572\n",
      "607\n",
      "632\n",
      "639\n",
      "652\n",
      "653\n",
      "654\n",
      "663\n",
      "664\n",
      "672\n",
      "680\n",
      "694\n",
      "708\n",
      "720\n",
      "737\n",
      "788\n",
      "807\n",
      "810\n",
      "836\n",
      "858\n",
      "861\n",
      "874\n",
      "953\n",
      "957\n",
      "972\n",
      "1075\n",
      "1084\n",
      "1097\n",
      "1098\n",
      "1099\n",
      "1108\n",
      "1131\n",
      "1154\n",
      "1164\n",
      "1176\n",
      "1184\n",
      "1190\n",
      "1199\n",
      "1257\n",
      "1260\n",
      "1279\n",
      "1287\n",
      "1306\n",
      "1329\n",
      "1338\n",
      "1339\n",
      "1362\n",
      "1381\n",
      "1382\n",
      "1395\n",
      "1400\n",
      "1401\n",
      "1402\n",
      "1403\n",
      "1405\n",
      "1421\n",
      "1422\n",
      "1423\n",
      "1431\n",
      "1434\n",
      "1435\n",
      "1437\n",
      "1444\n",
      "1498\n",
      "1501\n",
      "1508\n",
      "1510\n",
      "1511\n",
      "1513\n",
      "1515\n",
      "1520\n",
      "1525\n",
      "1526\n",
      "1529\n",
      "1530\n",
      "1535\n",
      "1541\n",
      "1545\n",
      "1546\n",
      "1552\n",
      "1559\n",
      "1560\n",
      "1561\n",
      "1595\n",
      "1597\n",
      "1607\n",
      "1627\n",
      "1628\n",
      "1634\n",
      "1644\n",
      "1654\n",
      "1657\n",
      "1660\n",
      "1661\n",
      "1685\n",
      "1686\n",
      "1696\n",
      "1701\n",
      "1705\n",
      "1718\n",
      "1723\n",
      "1739\n",
      "1752\n",
      "1759\n",
      "1779\n",
      "1786\n",
      "1790\n",
      "1795\n",
      "1810\n",
      "1811\n",
      "1812\n",
      "1838\n",
      "1852\n",
      "1861\n",
      "1862\n",
      "1863\n",
      "1881\n",
      "1882\n",
      "1886\n",
      "1887\n",
      "1914\n",
      "1916\n",
      "1918\n",
      "1928\n",
      "1933\n",
      "1936\n",
      "1941\n",
      "1945\n",
      "1967\n",
      "1968\n",
      "1990\n",
      "1993\n",
      "1994\n",
      "1997\n",
      "26000 28000\n",
      "14\n",
      "1\n",
      "9\n",
      "17\n",
      "45\n",
      "61\n",
      "65\n",
      "71\n",
      "84\n",
      "87\n",
      "88\n",
      "90\n",
      "91\n",
      "115\n",
      "127\n",
      "142\n",
      "157\n",
      "167\n",
      "175\n",
      "182\n",
      "210\n",
      "233\n",
      "255\n",
      "257\n",
      "273\n",
      "280\n",
      "306\n",
      "307\n",
      "365\n",
      "382\n",
      "422\n",
      "428\n",
      "440\n",
      "478\n",
      "486\n",
      "488\n",
      "490\n",
      "500\n",
      "502\n",
      "518\n",
      "519\n",
      "536\n",
      "545\n",
      "549\n",
      "602\n",
      "608\n",
      "609\n",
      "622\n",
      "651\n",
      "710\n",
      "717\n",
      "741\n",
      "743\n",
      "771\n",
      "780\n",
      "818\n",
      "820\n",
      "851\n",
      "862\n",
      "872\n",
      "900\n",
      "958\n",
      "1000\n",
      "1006\n",
      "1029\n",
      "1075\n",
      "1080\n",
      "1090\n",
      "1099\n",
      "1107\n",
      "1116\n",
      "1118\n",
      "1126\n",
      "1131\n",
      "1138\n",
      "1146\n",
      "1177\n",
      "1189\n",
      "1199\n",
      "1221\n",
      "1243\n",
      "1273\n",
      "1284\n",
      "1322\n",
      "1324\n",
      "1333\n",
      "1400\n",
      "1410\n",
      "1417\n",
      "1425\n",
      "1433\n",
      "1435\n",
      "1437\n",
      "1453\n",
      "1454\n",
      "1466\n",
      "1513\n",
      "1541\n",
      "1552\n",
      "1559\n",
      "1578\n",
      "1590\n",
      "1620\n",
      "1628\n",
      "1724\n",
      "1757\n",
      "1792\n",
      "1801\n",
      "1811\n",
      "1812\n",
      "1830\n",
      "1831\n",
      "1836\n",
      "1838\n",
      "1862\n",
      "1894\n",
      "1923\n",
      "1930\n",
      "1945\n",
      "1960\n",
      "1963\n",
      "1966\n",
      "1967\n",
      "1968\n",
      "28000 30000\n",
      "15\n",
      "1\n",
      "2\n",
      "3\n",
      "13\n",
      "49\n",
      "57\n",
      "65\n",
      "70\n",
      "114\n",
      "147\n",
      "151\n",
      "168\n",
      "193\n",
      "194\n",
      "195\n",
      "199\n",
      "205\n",
      "206\n",
      "218\n",
      "226\n",
      "241\n",
      "261\n",
      "307\n",
      "324\n",
      "353\n",
      "363\n",
      "369\n",
      "372\n",
      "422\n",
      "433\n",
      "436\n",
      "480\n",
      "498\n",
      "517\n",
      "551\n",
      "563\n",
      "572\n",
      "580\n",
      "581\n",
      "668\n",
      "680\n",
      "721\n",
      "722\n",
      "724\n",
      "739\n",
      "785\n",
      "791\n",
      "792\n",
      "798\n",
      "800\n",
      "802\n",
      "823\n",
      "847\n",
      "868\n",
      "870\n",
      "886\n",
      "908\n",
      "919\n",
      "935\n",
      "962\n",
      "967\n",
      "995\n",
      "1006\n",
      "1007\n",
      "1014\n",
      "1028\n",
      "1045\n",
      "1067\n",
      "1077\n",
      "1115\n",
      "1121\n",
      "1131\n",
      "1135\n",
      "1153\n",
      "1178\n",
      "1185\n",
      "1194\n",
      "1239\n",
      "1271\n",
      "1282\n",
      "1286\n",
      "1310\n",
      "1339\n",
      "1340\n",
      "1353\n",
      "1359\n",
      "1375\n",
      "1389\n",
      "1395\n",
      "1404\n",
      "1421\n",
      "1494\n",
      "1510\n",
      "1520\n",
      "1550\n",
      "1551\n",
      "1557\n",
      "1561\n",
      "1605\n",
      "1620\n",
      "1622\n",
      "1647\n",
      "1666\n",
      "1690\n",
      "1694\n",
      "1696\n",
      "1765\n",
      "1766\n",
      "1774\n",
      "1803\n",
      "1815\n",
      "1841\n",
      "1848\n",
      "1894\n",
      "1919\n",
      "1922\n",
      "1946\n",
      "1971\n",
      "30000 32000\n",
      "16\n",
      "1\n",
      "9\n",
      "16\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "48\n",
      "70\n",
      "82\n",
      "105\n",
      "111\n",
      "113\n",
      "166\n",
      "167\n",
      "172\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "195\n",
      "201\n",
      "203\n",
      "213\n",
      "235\n",
      "243\n",
      "244\n",
      "245\n",
      "257\n",
      "265\n",
      "269\n",
      "282\n",
      "290\n",
      "301\n",
      "323\n",
      "326\n",
      "328\n",
      "329\n",
      "333\n",
      "335\n",
      "340\n",
      "373\n",
      "377\n",
      "383\n",
      "384\n",
      "385\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "420\n",
      "439\n",
      "452\n",
      "456\n",
      "460\n",
      "467\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "485\n",
      "486\n",
      "525\n",
      "533\n",
      "535\n",
      "558\n",
      "561\n",
      "566\n",
      "604\n",
      "605\n",
      "612\n",
      "613\n",
      "614\n",
      "619\n",
      "641\n",
      "664\n",
      "799\n",
      "800\n",
      "827\n",
      "828\n",
      "829\n",
      "851\n",
      "857\n",
      "893\n",
      "1011\n",
      "1058\n",
      "1085\n",
      "1094\n",
      "1110\n",
      "1118\n",
      "1197\n",
      "1198\n",
      "1214\n",
      "1234\n",
      "1259\n",
      "1263\n",
      "1273\n",
      "1283\n",
      "1290\n",
      "1321\n",
      "1337\n",
      "1345\n",
      "1353\n",
      "1354\n",
      "1365\n",
      "1380\n",
      "1381\n",
      "1385\n",
      "1389\n",
      "1394\n",
      "1415\n",
      "1433\n",
      "1443\n",
      "1445\n",
      "1456\n",
      "1493\n",
      "1512\n",
      "1518\n",
      "1566\n",
      "1598\n",
      "1606\n",
      "1627\n",
      "1659\n",
      "1664\n",
      "1697\n",
      "1698\n",
      "1699\n",
      "1700\n",
      "1710\n",
      "1722\n",
      "1736\n",
      "1746\n",
      "1758\n",
      "1761\n",
      "1779\n",
      "1781\n",
      "1809\n",
      "1831\n",
      "1836\n",
      "1839\n",
      "1863\n",
      "1881\n",
      "1902\n",
      "1923\n",
      "1930\n",
      "1961\n",
      "1964\n",
      "1968\n",
      "1972\n",
      "32000 34000\n",
      "17\n",
      "1\n",
      "5\n",
      "6\n",
      "8\n",
      "11\n",
      "12\n",
      "15\n",
      "21\n",
      "22\n",
      "27\n",
      "31\n",
      "32\n",
      "34\n",
      "37\n",
      "40\n",
      "45\n",
      "46\n",
      "48\n",
      "49\n",
      "50\n",
      "62\n",
      "64\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "90\n",
      "109\n",
      "112\n",
      "118\n",
      "129\n",
      "138\n",
      "148\n",
      "152\n",
      "158\n",
      "159\n",
      "160\n",
      "163\n",
      "174\n",
      "231\n",
      "232\n",
      "257\n",
      "266\n",
      "270\n",
      "348\n",
      "371\n",
      "391\n",
      "400\n",
      "408\n",
      "411\n",
      "420\n",
      "427\n",
      "428\n",
      "432\n",
      "433\n",
      "440\n",
      "445\n",
      "461\n",
      "482\n",
      "508\n",
      "509\n",
      "510\n",
      "512\n",
      "515\n",
      "535\n",
      "555\n",
      "557\n",
      "558\n",
      "563\n",
      "568\n",
      "578\n",
      "583\n",
      "584\n",
      "592\n",
      "602\n",
      "631\n",
      "655\n",
      "668\n",
      "713\n",
      "746\n",
      "750\n",
      "778\n",
      "784\n",
      "787\n",
      "820\n",
      "843\n",
      "858\n",
      "927\n",
      "952\n",
      "958\n",
      "968\n",
      "971\n",
      "1005\n",
      "1013\n",
      "1018\n",
      "1041\n",
      "1047\n",
      "1063\n",
      "1071\n",
      "1087\n",
      "1100\n",
      "1101\n",
      "1136\n",
      "1193\n",
      "1222\n",
      "1232\n",
      "1235\n",
      "1241\n",
      "1260\n",
      "1271\n",
      "1280\n",
      "1285\n",
      "1293\n",
      "1320\n",
      "1349\n",
      "1383\n",
      "1396\n",
      "1397\n",
      "1446\n",
      "1501\n",
      "1576\n",
      "1661\n",
      "1675\n",
      "1688\n",
      "1703\n",
      "1709\n",
      "1711\n",
      "1726\n",
      "1739\n",
      "1742\n",
      "1757\n",
      "1830\n",
      "1855\n",
      "1872\n",
      "1885\n",
      "1894\n",
      "1904\n",
      "1920\n",
      "1934\n",
      "1953\n",
      "1954\n",
      "1962\n",
      "1982\n",
      "1985\n",
      "34000 36000\n",
      "18\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "54\n",
      "59\n",
      "77\n",
      "79\n",
      "115\n",
      "133\n",
      "135\n",
      "139\n",
      "163\n",
      "198\n",
      "199\n",
      "206\n",
      "229\n",
      "261\n",
      "284\n",
      "346\n",
      "423\n",
      "457\n",
      "473\n",
      "475\n",
      "484\n",
      "497\n",
      "505\n",
      "522\n",
      "545\n",
      "602\n",
      "606\n",
      "616\n",
      "618\n",
      "651\n",
      "652\n",
      "662\n",
      "669\n",
      "672\n",
      "685\n",
      "688\n",
      "704\n",
      "743\n",
      "750\n",
      "751\n",
      "754\n",
      "757\n",
      "784\n",
      "794\n",
      "839\n",
      "840\n",
      "844\n",
      "852\n",
      "858\n",
      "875\n",
      "962\n",
      "984\n",
      "985\n",
      "1007\n",
      "1040\n",
      "1049\n",
      "1061\n",
      "1063\n",
      "1092\n",
      "1106\n",
      "1125\n",
      "1147\n",
      "1153\n",
      "1160\n",
      "1194\n",
      "1231\n",
      "1250\n",
      "1258\n",
      "1269\n",
      "1290\n",
      "1316\n",
      "1317\n",
      "1318\n",
      "1337\n",
      "1338\n",
      "1339\n",
      "1340\n",
      "1341\n",
      "1342\n",
      "1343\n",
      "1344\n",
      "1345\n",
      "1346\n",
      "1347\n",
      "1348\n",
      "1349\n",
      "1350\n",
      "1351\n",
      "1352\n",
      "1353\n",
      "1354\n",
      "1355\n",
      "1356\n",
      "1357\n",
      "1358\n",
      "1359\n",
      "1416\n",
      "1465\n",
      "1484\n",
      "1494\n",
      "1508\n",
      "1512\n",
      "1529\n",
      "1537\n",
      "1564\n",
      "1572\n",
      "1577\n",
      "1594\n",
      "1645\n",
      "1685\n",
      "1687\n",
      "1726\n",
      "1744\n",
      "1788\n",
      "1806\n",
      "1856\n",
      "1870\n",
      "1891\n",
      "1900\n",
      "1941\n",
      "1942\n",
      "1943\n",
      "1950\n",
      "1951\n",
      "1952\n",
      "1953\n",
      "1954\n",
      "1955\n",
      "1956\n",
      "1957\n",
      "1987\n",
      "36000 38000\n",
      "19\n",
      "17\n",
      "78\n",
      "81\n",
      "110\n",
      "162\n",
      "163\n",
      "164\n",
      "211\n",
      "217\n",
      "218\n",
      "220\n",
      "228\n",
      "231\n",
      "233\n",
      "260\n",
      "296\n",
      "301\n",
      "303\n",
      "325\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "360\n",
      "366\n",
      "385\n",
      "388\n",
      "402\n",
      "451\n",
      "469\n",
      "475\n",
      "482\n",
      "534\n",
      "565\n",
      "566\n",
      "578\n",
      "579\n",
      "580\n",
      "581\n",
      "582\n",
      "583\n",
      "584\n",
      "599\n",
      "616\n",
      "619\n",
      "658\n",
      "660\n",
      "678\n",
      "679\n",
      "703\n",
      "709\n",
      "710\n",
      "719\n",
      "781\n",
      "805\n",
      "809\n",
      "810\n",
      "820\n",
      "834\n",
      "837\n",
      "867\n",
      "874\n",
      "878\n",
      "884\n",
      "901\n",
      "988\n",
      "1028\n",
      "1052\n",
      "1053\n",
      "1059\n",
      "1060\n",
      "1072\n",
      "1095\n",
      "1112\n",
      "1148\n",
      "1161\n",
      "1162\n",
      "1177\n",
      "1181\n",
      "1187\n",
      "1188\n",
      "1202\n",
      "1219\n",
      "1247\n",
      "1260\n",
      "1264\n",
      "1279\n",
      "1339\n",
      "1350\n",
      "1363\n",
      "1366\n",
      "1431\n",
      "1453\n",
      "1454\n",
      "1455\n",
      "1456\n",
      "1471\n",
      "1483\n",
      "1512\n",
      "1528\n",
      "1555\n",
      "1675\n",
      "1678\n",
      "1693\n",
      "1705\n",
      "1799\n",
      "1819\n",
      "1872\n",
      "1875\n",
      "1876\n",
      "1893\n",
      "1899\n",
      "1923\n",
      "1925\n",
      "1926\n",
      "1927\n",
      "1928\n",
      "1929\n",
      "1930\n",
      "1931\n",
      "1933\n",
      "1936\n",
      "1938\n",
      "1940\n",
      "1942\n",
      "1944\n",
      "1946\n",
      "1947\n",
      "1949\n",
      "1950\n",
      "1951\n",
      "1952\n",
      "1954\n",
      "1957\n",
      "1959\n",
      "1960\n",
      "1961\n",
      "1962\n",
      "1965\n",
      "1966\n",
      "1967\n",
      "1971\n",
      "1972\n",
      "1975\n",
      "1979\n",
      "1980\n",
      "1983\n",
      "1985\n",
      "1987\n",
      "1991\n",
      "1992\n",
      "1994\n",
      "1995\n"
     ]
    }
   ],
   "source": [
    "\"\"\"part = 20\n",
    "for i in range(38000, 38000, 2000):\n",
    "    print(i,i+2000)\n",
    "    print(part)\n",
    "    data = scrape_pages(list_urls[i:i+2000])\n",
    "    filename = 'carsNov'+str(part)+'.csv'\n",
    "    part+=1\n",
    "    data.to_csv(filename, index=False)\n",
    "    \"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}