01_Scrape_All_Links_gh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "\n",
    "from selenium import webdriver\n",
    "from datetime import datetime\n",
    "import time\n",
    "import pandas as pd\n",
    "import pickle as pk\n",
    "\n",
    "# Create an 'instance' of the driver.\n",
    "# This path should be to wherever you downloaded the driver.\n",
    "driver = webdriver.Chrome(executable_path=\"/Users/chris/chromedriver\")\n",
    "# A new Chrome (or other browser) window should open up.\n",
    "\n",
    "# Enter the main page, all condos in Bangkok are grouped by district.\n",
    "url ='https://www.hipflat.co.th/en/market/condo-bangkok-skik'\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write function to scrape all links from the webpage.\n",
    "\n",
    "def get_all_links(driver):\n",
    "    links = []\n",
    "    elements = driver.find_elements_by_class_name('directories__lists-element-name')\n",
    "    for elem in elements:\n",
    "        href = elem.get_attribute(\"href\")\n",
    "        links.append(href)\n",
    "    return links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run and store the links in 'district_links'\n",
    "\n",
    "start_time = datetime.now() \n",
    "\n",
    "district_links=get_all_links(driver)\n",
    "\n",
    "time_elapsed = datetime.now() - start_time \n",
    "print('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the length of 'district_links', there are 50 districts in Bangkok. Matched with wiki info.\n",
    "# https://en.wikipedia.org/wiki/List_of_districts_of_Bangkok\n",
    "\n",
    "len(district_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Re-run the function to retrive all condos in each district.\n",
    "# Append to 'condo_links'.\n",
    "\n",
    "start_time = datetime.now()\n",
    "condo_links=[]\n",
    "\n",
    "for dist in district_links:\n",
    "    print(len(condo_links),dist)\n",
    "    driver = webdriver.Chrome(executable_path=\"/Users/chris/chromedriver\")\n",
    "    driver.implicitly_wait(10)\n",
    "    driver.get(dist)\n",
    "    condo_links.append(get_all_links(driver))\n",
    "    driver.close()\n",
    "    time_elapsed = datetime.now() - start_time \n",
    "    print('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))\n",
    "print(\"completed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(condo_links))\n",
    "print(condo_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn a (nested) python list into a single list, that contains all the elements of sub lists\n",
    "# Named as 'condo_links_all'\n",
    "\n",
    "from itertools import chain\n",
    "condo_links_all=list(chain.from_iterable(condo_links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check total links in the 'condo_links_all' list. Total 2540 links.\n",
    "\n",
    "print(\"Total condos listed = \"+str(len(condo_links_all)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "condo_links_all[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump the retrived links to text file.\n",
    "\n",
    "with open(\"condo_links_all.txt\", \"w\") as f:\n",
    "    for s in condo_links_all:\n",
    "        f.write(str(s) +\"\\n\")\n",
    "print(\"completed\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}