Linkedin Jobs-v3 for recording.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from pandas.core.common import flatten\n",
    "import numpy as np\n",
    "import re\n",
    "import random\n",
    "from selenium import webdriver\n",
    "from selenium.common.exceptions import ElementClickInterceptedException\n",
    "# pressing certain keys\n",
    "# from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import pickle\n",
    "from datetime import datetime\n",
    "# using beautiful soup to find our data a bit easier\n",
    "# from bs4 import BeautifulSoup\n",
    "# also trying gazpacho soup\n",
    "from gazpacho import Soup\n",
    "# convert html to markdown\n",
    "from markdownify import markdownify as md\n",
    "# for US states\n",
    "import us"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# login via cookie\n",
    "BASE_URL = 'https://www.linkedin.com'\n",
    "def login_with_cookie(cookie_path):\n",
    "    # first load chromedriver\n",
    "    OPTIONS = webdriver.ChromeOptions()\n",
    "    # make the browser pretty thin so the page doesn't have multiple scroll bars\n",
    "    # width,height\n",
    "    OPTIONS.add_argument(\"window-size=600,800\")\n",
    "    OPTIONS.headless = True\n",
    "    # remove all these when watching it the first time\n",
    "    # if we need to be a realer browser (not a headless chrome)\n",
    "    print('opening chrome')\n",
    "    driver = webdriver.Chrome('chromedriver',options=OPTIONS)\n",
    "    # login with cookie, how long is cookie timeout? unknown\n",
    "    # think about logging in via username and pass\n",
    "    COOKIE_NAME = 'linkedin_cookies_2021-02-14.pkl'\n",
    "    # load the page without the cookie\n",
    "    driver.get(BASE_URL)\n",
    "    # load the cookie\n",
    "    with open(cookie_path, \"rb\") as f:\n",
    "        cookies = pickle.load(f)  \n",
    "    # cookies = pickle.load(open('meetup_cookies.pkl','rb'))    \n",
    "    print('loading cookies')\n",
    "    for cookie in cookies:\n",
    "        driver.add_cookie(cookie)\n",
    "    # load the page again after the cookie is loaded, this is the same as logging in\n",
    "    print('opening chrome with cookies to login')\n",
    "    driver.get(BASE_URL)\n",
    "    # minimise the messages area\n",
    "    minimise_msgs(driver)\n",
    "    return driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# minimise the messaging bit if it's open\n",
    "def minimise_msgs(driver):\n",
    "    # if this is open\n",
    "    msg_min = len(driver.find_elements_by_class_name('msg-overlay-list-bubble--is-minimized'))\n",
    "    if msg_min == 0:\n",
    "        try:\n",
    "            print('minimising the messages overlay')\n",
    "            msg_buttons = driver.find_elements_by_class_name('msg-overlay-bubble-header__control')\n",
    "            msg_buttons[1].click()\n",
    "        except:\n",
    "            pass\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we scroll in chunks because things don't load if we scroll to the bottom immediately\n",
    "def scroll_to_bottom2(driver,scroll_num):\n",
    "    pause = 2 # one sec is def too fast, script starts to fail to find the next page button\n",
    "    while(scroll_num>0):\n",
    "        print('scrolling down 800 pixels')\n",
    "        driver.execute_script(\"window.scrollBy(0,800);\")\n",
    "        time.sleep(pause)\n",
    "        scroll_num -= 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# safest to get the loc manually via the URL\n",
    "# e.g. location=Sydney%2C%20New%20South%20Wales%2C%20Australia\n",
    "# first run basically finds the pagination -- fix the pagination.. look at the last number and make that the last page\n",
    "# next loop grabs all job links\n",
    "def search_grab_links_v3(driver,kws,loc,top_pages='1'):\n",
    "    kws = kws.replace(' ','%20').replace(',','%2C')\n",
    "    URL = BASE_URL + '/jobs/search/?keywords='+kws+'&'+'location='+loc\n",
    "    # get the search engine results page (SERP)\n",
    "    driver.get(URL)\n",
    "    # add a sleep time, so the page is fully loaded before we scroll\n",
    "#     time.sleep(3)\n",
    "    # find the minimise messaging button, there's only 2 with this class\n",
    "    # the first is the write new message button\n",
    "    # only need to do this once because we're in the same session\n",
    "    minimise_msgs(driver)\n",
    "    # scroll to the bottom to load all the jobs, linkedin takes about 5x scrolls of 800px\n",
    "    scroll_to_bottom2(driver,scroll_num=5)\n",
    "    # grab the page's html as a string\n",
    "    html = driver.page_source\n",
    "    soup = Soup(html)\n",
    "    # find total pages we have\n",
    "    # driver.find_elements_by_xpath('//button[contains(@aria-label,\"Page\")]')[-1].text\n",
    "    pagi = soup.find('li', {'class': 'pagination'}, partial=True)[-1].find('span').text\n",
    "    # find all the job links anchor tags\n",
    "    job_links1 = soup.find('a', {'class':'job-card-list__title'})\n",
    "    # find all href links in the a tags, and cut them to minimum size /jobs/view/123456789/\n",
    "    try:\n",
    "        job_links = [re.search('^(.+)\\?',x.attrs['href']).group(1) for x in job_links1]\n",
    "    except Exception as e:\n",
    "        job_links = ['nolink']\n",
    "        print(f'Exception is {e}')\n",
    "    # put the job links in our all_job_links list\n",
    "    all_job_links = []\n",
    "    all_job_links.append(job_links)\n",
    "    \n",
    "    # there's a chance there could be say 20 pages, and we don't want to wait this long\n",
    "    if(top_pages == '1'):\n",
    "        return all_job_links\n",
    "    # pages is always a str for both below scenarios\n",
    "    elif(top_pages=='all'):\n",
    "        pages = pagi\n",
    "    else:\n",
    "        pages = top_pages\n",
    "    \n",
    "    # loop covers grabbing job links\n",
    "    for i in range(2,int(pages)+1):\n",
    "        # click the next page button, will click 2, since we're on 1 first\n",
    "        try:\n",
    "            print(f'clicking page button {i}')\n",
    "#             driver.find_element_by_css_selector(f\"button[aria-label='Page {i}']\").click()\n",
    "            driver.find_element_by_xpath(f\"//button/span[contains(text(), '{i}')]/ancestor::button\").click()\n",
    "        except Exception as e:\n",
    "            # If pop-up overlay appears, click the X button to close\n",
    "            time.sleep(2) # Sometimes the pop-up takes time to load\n",
    "            print(f'error {e}, retrying clicking page button {i} with …')\n",
    "            driver.find_element_by_xpath(f\"//button/span[contains(text(), '…')]/ancestor::button\").click()\n",
    "#             driver.find_element_by_css_selector(f\"button[aria-label='Page {i}']\").click()\n",
    "        # wait for site to load\n",
    "        time.sleep(1)\n",
    "        # scroll to bottom\n",
    "        scroll_to_bottom2(driver,scroll_num=5)\n",
    "        # grab the page source\n",
    "        html = driver.page_source\n",
    "        # make it a soup\n",
    "        soup = Soup(html)\n",
    "        # find total pages we have\n",
    "#         pagi = soup.find('li', {'class': 'pagination'}, partial=True)[-1].find('span').text\n",
    "        # find all the job links anchor tags\n",
    "        job_links1 = soup.find('a', {'class':'job-card-list__title'})\n",
    "        try:\n",
    "            job_links = [re.search('^(.+)\\?',x.attrs['href']).group(1) for x in job_links1]\n",
    "        except Exception as e:\n",
    "            job_links = ['nolink']\n",
    "            print(f'Exception is {e}')\n",
    "        # put the job links in our all_job_links list\n",
    "        all_job_links.append(job_links)\n",
    "    \n",
    "    # all_job_links is a list of lists, this last bit flattens it into a single list\n",
    "    return list(flatten(all_job_links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# safest to get the loc manually via the URL\n",
    "# e.g. location=Sydney%2C%20New%20South%20Wales%2C%20Australia\n",
    "# first run basically finds the pagination -- fix the pagination.. look at the last number and make that the last page\n",
    "# next loop grabs all job links\n",
    "def search_grab_links_v2(driver,kws,loc,top_pages='1'):\n",
    "    kws = kws.replace(' ','%20').replace(',','%2C')\n",
    "    URL = BASE_URL + '/jobs/search/?keywords='+kws+'&'+'location='+loc\n",
    "    # get the search engine results page (SERP)\n",
    "    driver.get(URL)\n",
    "    # add a sleep time, so the page is fully loaded before we scroll\n",
    "#     time.sleep(3)\n",
    "    # find the minimise messaging button, there's only 2 with this class\n",
    "    # the first is the write new message button\n",
    "    # only need to do this once because we're in the same session\n",
    "    minimise_msgs(driver)\n",
    "    # scroll to the bottom to load all the jobs, linkedin takes about 5x scrolls of 800px\n",
    "    scroll_to_bottom2(driver,scroll_num=5)\n",
    "    # grab the page's html as a string\n",
    "    html = driver.page_source\n",
    "    soup = Soup(html)\n",
    "    # find total pages we have\n",
    "    # driver.find_elements_by_xpath('//button[contains(@aria-label,\"Page\")]')[-1].text\n",
    "    pagi = soup.find('li', {'class': 'pagination'}, partial=True)[-1].find('span').text\n",
    "    # find all the job links anchor tags\n",
    "    job_links1 = soup.find('a', {'class':'job-card-list__title'})\n",
    "    # find all href links in the a tags, and cut them to minimum size /jobs/view/123456789/\n",
    "    try:\n",
    "        job_links = [re.search('^(.+)\\?',x.attrs['href']).group(1) for x in job_links1]\n",
    "    except Exception as e:\n",
    "        job_links = ['nolink']\n",
    "        print(f'Exception is {e}')\n",
    "    # put the job links in our all_job_links list\n",
    "    all_job_links = []\n",
    "    all_job_links.append(job_links)\n",
    "    \n",
    "    # there's a chance there could be say 20 pages, and we don't want to wait this long\n",
    "    if(top_pages == '1'):\n",
    "        return all_job_links\n",
    "    # pages is always a str for both below scenarios\n",
    "    elif(top_pages=='all'):\n",
    "        pages = pagi\n",
    "    else:\n",
    "        pages = top_pages\n",
    "    \n",
    "    # loop covers grabbing job links\n",
    "    for i in range(2,int(pages)+1):\n",
    "        # click the next page button, will click 2, since we're on 1 first\n",
    "        try:\n",
    "            print(f'clicking page button {i}')\n",
    "            driver.find_element_by_css_selector(f\"button[aria-label='Page {i}']\").click()\n",
    "        except Exception as e:\n",
    "            # If pop-up overlay appears, click the X button to close\n",
    "            time.sleep(2) # Sometimes the pop-up takes time to load\n",
    "            print(f'error {e}, retrying clicking page button {i}')\n",
    "            driver.find_element_by_css_selector(f\"button[aria-label='Page {i}']\").click()\n",
    "        # wait for site to load\n",
    "        time.sleep(1)\n",
    "        # scroll to bottom\n",
    "        scroll_to_bottom2(driver,scroll_num=5)\n",
    "        # grab the page source\n",
    "        html = driver.page_source\n",
    "        # make it a soup\n",
    "        soup = Soup(html)\n",
    "        # find total pages we have\n",
    "#         pagi = soup.find('li', {'class': 'pagination'}, partial=True)[-1].find('span').text\n",
    "        # find all the job links anchor tags\n",
    "        job_links1 = soup.find('a', {'class':'job-card-list__title'})\n",
    "        try:\n",
    "            job_links = [re.search('^(.+)\\?',x.attrs['href']).group(1) for x in job_links1]\n",
    "        except Exception as e:\n",
    "            job_links = ['nolink']\n",
    "            print(f'Exception is {e}')\n",
    "            \n",
    "        # put the job links in our all_job_links list\n",
    "        all_job_links.append(job_links)\n",
    "    \n",
    "    # all_job_links is a list of lists, this last bit flattens it into a single list\n",
    "    return list(flatten(all_job_links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use some dirty method of cleaning this html so we can run NLP on it later\n",
    "def soup_to_md(soup):\n",
    "    # remove the divs that surround, gazpacho doesn't seem to have a way to select all\n",
    "    temp1 = re.search('\\n(.+)\\n',str(soup)).group(1)\n",
    "    # make it markdown to remove other tags\n",
    "    temp2 = md(temp1)\n",
    "    # remove \\n created by markdown\n",
    "    temp3 = temp2.replace('\\n','')\n",
    "    # remove *'s created by markdown\n",
    "    temp4 = temp3.replace('*','')\n",
    "    # replace >1 space with just 1 space\n",
    "    temp5 = re.sub(' +',' ',temp4)\n",
    "    return temp5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5 sec x 100 = 500 sec ≈ 8.3min\n",
    "# 5 sec x 150 = 750 sec ≈ 12.5min\n",
    "# 5 sec x 200 = 1000 sec ≈ 16.6min\n",
    "# make the script to follow each of these links and grab the nice text\n",
    "def grab_job_descs_v2(links):\n",
    "    total_links = len(links)\n",
    "    out = []\n",
    "    \n",
    "    for i,link in enumerate(links):\n",
    "        try:\n",
    "            print(f'grabbing link {link}, {i+1} of {total_links}')\n",
    "            out.append(grab_desc_v2(link))\n",
    "            time.sleep(5)\n",
    "        except:\n",
    "            print(f'this link failed {link}')\n",
    "            \n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean this job details bit\n",
    "def grab_job_details(soup):\n",
    "    job_extra_keys_raw = soup.find('li', {'class': 'job-criteria'}, partial=True, mode='all')\n",
    "    job_extra_vals_raw = soup.find('span', {'class': 'job-criteria'}, partial=True, mode='all')\n",
    "    \n",
    "    # some logic based on number of list items found\n",
    "    num_keys = len(job_extra_keys_raw)\n",
    "    if num_keys == 0 :\n",
    "        return {}\n",
    "    if num_keys == 1 :\n",
    "        key = job_extra_keys_raw[0].text.title()\n",
    "        val = job_extra_vals_raw[0].text\n",
    "        return {key:val}\n",
    "    else :\n",
    "        job_extra_keys = [key.text.title() for key in job_extra_keys_raw]\n",
    "        job_extra_vals = [val.text for val in job_extra_vals_raw]\n",
    "        return dict(zip(job_extra_keys,job_extra_vals))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# what we do here is odd, this request is without login\n",
    "# this is to preserve some kind of purity in your logged in account\n",
    "# one consequence of this is that some jobs might not be public\n",
    "# 2nd is that css and html is actually different for logged in vs public\n",
    "def grab_desc_v2(link):\n",
    "    BASE_URL = 'https://www.linkedin.com'\n",
    "    URL = BASE_URL + link \n",
    "    res = requests.get(url=URL)\n",
    "    # using this gazpacho branded Soup\n",
    "    soup = Soup(res.text)\n",
    "    # they all live in these *topcard* classes\n",
    "    # job_card contains the organisation and location\n",
    "    # job_title lives above the card in the h1\n",
    "    job_card = soup.find('span', {'class': 'topcard'}, partial=True)\n",
    "    job_title = soup.find('h1', {'class': 'topcard'}, partial=True).text\n",
    "    job_org = job_card[0].text\n",
    "    job_loc = job_card[1].text\n",
    "    # grab job description\n",
    "    soup_desc = soup.find('div', {'class': 'show-more-less'}, partial=True)\n",
    "    job_desc = soup_to_md(soup_desc)\n",
    "    # grab job extra details\n",
    "    # there is a variable amount of these, sometimes there might be none\n",
    "    # so we offshore our cleaning\n",
    "    job_extra_details = grab_job_details(soup)\n",
    "    return {\n",
    "        'Job Title':job_title,\n",
    "        'Company':job_org,\n",
    "        'Job Location':job_loc,\n",
    "        'Job URL':URL,\n",
    "        'Job Description':job_desc,\n",
    "        'Job Details':job_extra_details\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# job list to df\n",
    "def joblist_to_df(listt):\n",
    "    # make the df\n",
    "    df1 = pd.DataFrame(listt)\n",
    "    # json normalize the extra details\n",
    "    df2 = pd.json_normalize(df1['Job Details'])\n",
    "    # append extra details\n",
    "    df3 = pd.concat([df1,df2],axis='columns')\n",
    "    # remove the old extra details column\n",
    "    df = df3.drop(columns=['Job Details'])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opening chrome\n",
      "loading cookies\n",
      "opening chrome with cookies to login\n",
      "minimising the messages overlay\n",
      "minimising the messages overlay\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 2\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 3\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 4\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 5\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 6\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 7\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 8\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 9\n",
      "error Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//button/span[contains(text(), '9')]/ancestor::button\"}\n",
      "  (Session info: headless chrome=92.0.4515.107)\n",
      ", retrying clicking page button 9 with …\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 10\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 11\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 12\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 13\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 14\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 15\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 16\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 17\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 18\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 19\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 20\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 21\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 22\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 23\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 24\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 25\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 26\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 27\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 28\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 29\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 30\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 31\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 32\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 33\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 34\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 35\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "clicking page button 36\n",
      "error Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//button/span[contains(text(), '36')]/ancestor::button\"}\n",
      "  (Session info: headless chrome=92.0.4515.107)\n",
      ", retrying clicking page button 36 with …\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "scrolling down 800 pixels\n",
      "grabbing link /jobs/view/2622582377/, 1 of 899\n",
      "grabbing link /jobs/view/2649187443/, 2 of 899\n",
      "grabbing link /jobs/view/2652778606/, 3 of 899\n",
      "grabbing link /jobs/view/2653604906/, 4 of 899\n",
      "this link failed /jobs/view/2653604906/\n",
      "grabbing link /jobs/view/2661212338/, 5 of 899\n",
      "grabbing link /jobs/view/2661268390/, 6 of 899\n",
      "grabbing link /jobs/view/2652779390/, 7 of 899\n",
      "grabbing link /jobs/view/2641013502/, 8 of 899\n",
      "this link failed /jobs/view/2641013502/\n",
      "grabbing link /jobs/view/2618292462/, 9 of 899\n",
      "grabbing link /jobs/view/2643915638/, 10 of 899\n",
      "grabbing link /jobs/view/2652776926/, 11 of 899\n",
      "grabbing link /jobs/view/2661267536/, 12 of 899\n",
      "this link failed /jobs/view/2661267536/\n",
      "grabbing link /jobs/view/2659531311/, 13 of 899\n",
      "grabbing link /jobs/view/2643915639/, 14 of 899\n",
      "grabbing link /jobs/view/2641016457/, 15 of 899\n",
      "grabbing link /jobs/view/2634614981/, 16 of 899\n",
      "this link failed /jobs/view/2634614981/\n",
      "grabbing link /jobs/view/2645644235/, 17 of 899\n",
      "grabbing link /jobs/view/2526396205/, 18 of 899\n",
      "this link failed /jobs/view/2526396205/\n",
      "grabbing link /jobs/view/2661215024/, 19 of 899\n",
      "this link failed /jobs/view/2661215024/\n",
      "grabbing link /jobs/view/2661270361/, 20 of 899\n",
      "grabbing link /jobs/view/2664975567/, 21 of 899\n",
      "this link failed /jobs/view/2664975567/\n",
      "grabbing link /jobs/view/2538165891/, 22 of 899\n",
      "grabbing link /jobs/view/2618293338/, 23 of 899\n",
      "grabbing link /jobs/view/2599194340/, 24 of 899\n",
      "this link failed /jobs/view/2599194340/\n",
      "grabbing link /jobs/view/2652775969/, 25 of 899\n",
      "grabbing link /jobs/view/2652775968/, 26 of 899\n",
      "this link failed /jobs/view/2652775968/\n",
      "grabbing link /jobs/view/2658255684/, 27 of 899\n",
      "this link failed /jobs/view/2658255684/\n",
      "grabbing link /jobs/view/2656848457/, 28 of 899\n",
      "this link failed /jobs/view/2656848457/\n",
      "grabbing link /jobs/view/2655577744/, 29 of 899\n",
      "grabbing link /jobs/view/2659529600/, 30 of 899\n",
      "this link failed /jobs/view/2659529600/\n",
      "grabbing link /jobs/view/2667354081/, 31 of 899\n",
      "grabbing link /jobs/view/2654017980/, 32 of 899\n",
      "this link failed /jobs/view/2654017980/\n",
      "grabbing link /jobs/view/2553553692/, 33 of 899\n",
      "this link failed /jobs/view/2553553692/\n",
      "grabbing link /jobs/view/2642030195/, 34 of 899\n",
      "grabbing link /jobs/view/2659588090/, 35 of 899\n",
      "this link failed /jobs/view/2659588090/\n",
      "grabbing link /jobs/view/2652927144/, 36 of 899\n",
      "this link failed /jobs/view/2652927144/\n",
      "grabbing link /jobs/view/2661267541/, 37 of 899\n",
      "grabbing link /jobs/view/2656328915/, 38 of 899\n",
      "grabbing link /jobs/view/2642031060/, 39 of 899\n",
      "this link failed /jobs/view/2642031060/\n",
      "grabbing link /jobs/view/2658295750/, 40 of 899\n",
      "grabbing link /jobs/view/2658294973/, 41 of 899\n",
      "this link failed /jobs/view/2658294973/\n",
      "grabbing link /jobs/view/2659123236/, 42 of 899\n",
      "this link failed /jobs/view/2659123236/\n",
      "grabbing link /jobs/view/2642388000/, 43 of 899\n",
      "grabbing link /jobs/view/2661922548/, 44 of 899\n",
      "this link failed /jobs/view/2661922548/\n",
      "grabbing link /jobs/view/2662625113/, 45 of 899\n",
      "grabbing link /jobs/view/2656848508/, 46 of 899\n",
      "grabbing link /jobs/view/2643919129/, 47 of 899\n",
      "grabbing link /jobs/view/2621623033/, 48 of 899\n",
      "this link failed /jobs/view/2621623033/\n",
      "grabbing link /jobs/view/2652775971/, 49 of 899\n",
      "grabbing link /jobs/view/2654207003/, 50 of 899\n",
      "grabbing link /jobs/view/2643917373/, 51 of 899\n",
      "grabbing link /jobs/view/2648907636/, 52 of 899\n",
      "grabbing link /jobs/view/2655787916/, 53 of 899\n",
      "grabbing link /jobs/view/2657364525/, 54 of 899\n",
      "grabbing link /jobs/view/2661260575/, 55 of 899\n",
      "grabbing link /jobs/view/2661212402/, 56 of 899\n",
      "this link failed /jobs/view/2661212402/\n",
      "grabbing link /jobs/view/2661266559/, 57 of 899\n",
      "this link failed /jobs/view/2661266559/\n",
      "grabbing link /jobs/view/2652937907/, 58 of 899\n",
      "grabbing link /jobs/view/2659883815/, 59 of 899\n",
      "this link failed /jobs/view/2659883815/\n",
      "grabbing link /jobs/view/2667363550/, 60 of 899\n",
      "grabbing link /jobs/view/2660737945/, 61 of 899\n",
      "grabbing link /jobs/view/2661290231/, 62 of 899\n",
      "grabbing link /jobs/view/2653641392/, 63 of 899\n",
      "this link failed /jobs/view/2653641392/\n",
      "grabbing link /jobs/view/2663919199/, 64 of 899\n",
      "this link failed /jobs/view/2663919199/\n",
      "grabbing link /jobs/view/2661204364/, 65 of 899\n",
      "grabbing link /jobs/view/2626666560/, 66 of 899\n",
      "this link failed /jobs/view/2626666560/\n",
      "grabbing link /jobs/view/2664546316/, 67 of 899\n",
      "this link failed /jobs/view/2664546316/\n",
      "grabbing link /jobs/view/2646738152/, 68 of 899\n",
      "grabbing link /jobs/view/2637614769/, 69 of 899\n",
      "grabbing link /jobs/view/2600965496/, 70 of 899\n",
      "grabbing link /jobs/view/2661290232/, 71 of 899\n",
      "this link failed /jobs/view/2661290232/\n",
      "grabbing link /jobs/view/2640336478/, 72 of 899\n",
      "grabbing link /jobs/view/2658232026/, 73 of 899\n",
      "this link failed /jobs/view/2658232026/\n",
      "grabbing link /jobs/view/2654881893/, 74 of 899\n",
      "this link failed /jobs/view/2654881893/\n",
      "grabbing link /jobs/view/2650989613/, 75 of 899\n",
      "grabbing link /jobs/view/2600684241/, 76 of 899\n",
      "grabbing link /jobs/view/2655726790/, 77 of 899\n",
      "grabbing link /jobs/view/2640390570/, 78 of 899\n",
      "grabbing link /jobs/view/2661235568/, 79 of 899\n",
      "grabbing link /jobs/view/2631344135/, 80 of 899\n",
      "this link failed /jobs/view/2631344135/\n",
      "grabbing link /jobs/view/2663529227/, 81 of 899\n",
      "grabbing link /jobs/view/2650583794/, 82 of 899\n",
      "this link failed /jobs/view/2650583794/\n",
      "grabbing link /jobs/view/2655789636/, 83 of 899\n",
      "grabbing link /jobs/view/2654011578/, 84 of 899\n",
      "this link failed /jobs/view/2654011578/\n",
      "grabbing link /jobs/view/2662540522/, 85 of 899\n",
      "grabbing link /jobs/view/2661969117/, 86 of 899\n",
      "grabbing link /jobs/view/2642715318/, 87 of 899\n",
      "grabbing link /jobs/view/2650983303/, 88 of 899\n",
      "this link failed /jobs/view/2650983303/\n",
      "grabbing link /jobs/view/2639275575/, 89 of 899\n",
      "this link failed /jobs/view/2639275575/\n",
      "grabbing link /jobs/view/2666159229/, 90 of 899\n",
      "this link failed /jobs/view/2666159229/\n",
      "grabbing link /jobs/view/2655795374/, 91 of 899\n",
      "this link failed /jobs/view/2655795374/\n",
      "grabbing link /jobs/view/2655701485/, 92 of 899\n",
      "grabbing link /jobs/view/2656360474/, 93 of 899\n",
      "grabbing link /jobs/view/2658230176/, 94 of 899\n",
      "grabbing link /jobs/view/2656971870/, 95 of 899\n",
      "grabbing link /jobs/view/2661945436/, 96 of 899\n",
      "this link failed /jobs/view/2661945436/\n",
      "grabbing link /jobs/view/2658239505/, 97 of 899\n",
      "this link failed /jobs/view/2658239505/\n",
      "grabbing link /jobs/view/2637633872/, 98 of 899\n",
      "grabbing link /jobs/view/2658923030/, 99 of 899\n",
      "this link failed /jobs/view/2658923030/\n",
      "grabbing link /jobs/view/2646761972/, 100 of 899\n",
      "grabbing link /jobs/view/2634863514/, 101 of 899\n",
      "grabbing link /jobs/view/2656341547/, 102 of 899\n",
      "this link failed /jobs/view/2656341547/\n",
      "grabbing link /jobs/view/2650048580/, 103 of 899\n",
      "grabbing link /jobs/view/2642089253/, 104 of 899\n",
      "this link failed /jobs/view/2642089253/\n",
      "grabbing link /jobs/view/2564935258/, 105 of 899\n",
      "this link failed /jobs/view/2564935258/\n",
      "grabbing link /jobs/view/2653665722/, 106 of 899\n",
      "grabbing link /jobs/view/2658890144/, 107 of 899\n",
      "this link failed /jobs/view/2658890144/\n",
      "grabbing link /jobs/view/2657360853/, 108 of 899\n",
      "grabbing link /jobs/view/2661464868/, 109 of 899\n",
      "grabbing link /jobs/view/2575060292/, 110 of 899\n",
      "grabbing link /jobs/view/2581868561/, 111 of 899\n",
      "this link failed /jobs/view/2581868561/\n",
      "grabbing link /jobs/view/2648505043/, 112 of 899\n",
      "this link failed /jobs/view/2648505043/\n",
      "grabbing link /jobs/view/2631292437/, 113 of 899\n",
      "grabbing link /jobs/view/2603147594/, 114 of 899\n",
      "grabbing link /jobs/view/2654887104/, 115 of 899\n",
      "this link failed /jobs/view/2654887104/\n",
      "grabbing link /jobs/view/2662948875/, 116 of 899\n",
      "grabbing link /jobs/view/2576822379/, 117 of 899\n",
      "grabbing link /jobs/view/2652754311/, 118 of 899\n",
      "grabbing link /jobs/view/2659869019/, 119 of 899\n",
      "grabbing link /jobs/view/2656843223/, 120 of 899\n",
      "grabbing link /jobs/view/2646196650/, 121 of 899\n",
      "grabbing link /jobs/view/2661255472/, 122 of 899\n",
      "grabbing link /jobs/view/2655740306/, 123 of 899\n",
      "grabbing link /jobs/view/2645660939/, 124 of 899\n",
      "this link failed /jobs/view/2645660939/\n",
      "grabbing link /jobs/view/2655789930/, 125 of 899\n",
      "grabbing link /jobs/view/2655746538/, 126 of 899\n",
      "this link failed /jobs/view/2655746538/\n",
      "grabbing link /jobs/view/2635357303/, 127 of 899\n",
      "this link failed /jobs/view/2635357303/\n",
      "grabbing link /jobs/view/2642089609/, 128 of 899\n",
      "grabbing link /jobs/view/2649959855/, 129 of 899\n",
      "this link failed /jobs/view/2649959855/\n",
      "grabbing link /jobs/view/2656811739/, 130 of 899\n",
      "this link failed /jobs/view/2656811739/\n",
      "grabbing link /jobs/view/2658214165/, 131 of 899\n",
      "grabbing link /jobs/view/2654888083/, 132 of 899\n",
      "grabbing link /jobs/view/2621616838/, 133 of 899\n",
      "grabbing link /jobs/view/2654436968/, 134 of 899\n",
      "grabbing link /jobs/view/2655880117/, 135 of 899\n",
      "this link failed /jobs/view/2655880117/\n",
      "grabbing link /jobs/view/2630738206/, 136 of 899\n",
      "this link failed /jobs/view/2630738206/\n",
      "grabbing link /jobs/view/2642779637/, 137 of 899\n",
      "grabbing link /jobs/view/2656309185/, 138 of 899\n",
      "this link failed /jobs/view/2656309185/\n",
      "grabbing link /jobs/view/2658882000/, 139 of 899\n",
      "this link failed /jobs/view/2658882000/\n",
      "grabbing link /jobs/view/2641052654/, 140 of 899\n",
      "grabbing link /jobs/view/2642784007/, 141 of 899\n",
      "grabbing link /jobs/view/2601607160/, 142 of 899\n",
      "grabbing link /jobs/view/2645681916/, 143 of 899\n",
      "grabbing link /jobs/view/2653653119/, 144 of 899\n",
      "grabbing link /jobs/view/2630763655/, 145 of 899\n",
      "grabbing link /jobs/view/2661792199/, 146 of 899\n",
      "grabbing link /jobs/view/2659912685/, 147 of 899\n",
      "this link failed /jobs/view/2659912685/\n",
      "grabbing link /jobs/view/2661718308/, 148 of 899\n",
      "grabbing link /jobs/view/2638867849/, 149 of 899\n",
      "this link failed /jobs/view/2638867849/\n",
      "grabbing link /jobs/view/2658142189/, 150 of 899\n",
      "grabbing link /jobs/view/2638883602/, 151 of 899\n",
      "grabbing link /jobs/view/2619848212/, 152 of 899\n",
      "grabbing link /jobs/view/2626664637/, 153 of 899\n",
      "grabbing link /jobs/view/2642082057/, 154 of 899\n",
      "grabbing link /jobs/view/2641088887/, 155 of 899\n",
      "grabbing link /jobs/view/2663175952/, 156 of 899\n",
      "this link failed /jobs/view/2663175952/\n",
      "grabbing link /jobs/view/2654609629/, 157 of 899\n",
      "grabbing link /jobs/view/2656810889/, 158 of 899\n",
      "grabbing link /jobs/view/2547586449/, 159 of 899\n",
      "this link failed /jobs/view/2547586449/\n",
      "grabbing link /jobs/view/2656305868/, 160 of 899\n",
      "this link failed /jobs/view/2656305868/\n",
      "grabbing link /jobs/view/2667358786/, 161 of 899\n",
      "this link failed /jobs/view/2667358786/\n",
      "grabbing link /jobs/view/2656602662/, 162 of 899\n",
      "grabbing link /jobs/view/2601085164/, 163 of 899\n",
      "this link failed /jobs/view/2601085164/\n",
      "grabbing link /jobs/view/2606489923/, 164 of 899\n",
      "grabbing link /jobs/view/2651907833/, 165 of 899\n",
      "this link failed /jobs/view/2651907833/\n",
      "grabbing link /jobs/view/2658827753/, 166 of 899\n",
      "this link failed /jobs/view/2658827753/\n",
      "grabbing link /jobs/view/2636898677/, 167 of 899\n",
      "grabbing link /jobs/view/2659866249/, 168 of 899\n",
      "grabbing link /jobs/view/2647593963/, 169 of 899\n",
      "grabbing link /jobs/view/2630741606/, 170 of 899\n",
      "this link failed /jobs/view/2630741606/\n",
      "grabbing link /jobs/view/2653670235/, 171 of 899\n",
      "grabbing link /jobs/view/2664166405/, 172 of 899\n",
      "grabbing link /jobs/view/2653664964/, 173 of 899\n",
      "grabbing link /jobs/view/2591565241/, 174 of 899\n",
      "grabbing link /jobs/view/2650918616/, 175 of 899\n",
      "grabbing link /jobs/view/2636000585/, 176 of 899\n",
      "this link failed /jobs/view/2636000585/\n",
      "grabbing link /jobs/view/2576824577/, 177 of 899\n",
      "grabbing link /jobs/view/2658874580/, 178 of 899\n",
      "this link failed /jobs/view/2658874580/\n",
      "grabbing link /jobs/view/2640369675/, 179 of 899\n",
      "grabbing link /jobs/view/2650060018/, 180 of 899\n",
      "this link failed /jobs/view/2650060018/\n",
      "grabbing link /jobs/view/2652775096/, 181 of 899\n",
      "grabbing link /jobs/view/2661275291/, 182 of 899\n",
      "grabbing link /jobs/view/2649153609/, 183 of 899\n",
      "grabbing link /jobs/view/2645120849/, 184 of 899\n",
      "grabbing link /jobs/view/2596240859/, 185 of 899\n",
      "this link failed /jobs/view/2596240859/\n",
      "grabbing link /jobs/view/2655754960/, 186 of 899\n",
      "grabbing link /jobs/view/2571285167/, 187 of 899\n",
      "grabbing link /jobs/view/2656815432/, 188 of 899\n",
      "grabbing link /jobs/view/2663048300/, 189 of 899\n",
      "this link failed /jobs/view/2663048300/\n",
      "grabbing link /jobs/view/2646239496/, 190 of 899\n",
      "grabbing link /jobs/view/2656460204/, 191 of 899\n",
      "this link failed /jobs/view/2656460204/\n",
      "grabbing link /jobs/view/2660097265/, 192 of 899\n",
      "this link failed /jobs/view/2660097265/\n",
      "grabbing link /jobs/view/2647345888/, 193 of 899\n",
      "grabbing link /jobs/view/2655881073/, 194 of 899\n",
      "this link failed /jobs/view/2655881073/\n",
      "grabbing link /jobs/view/2627213079/, 195 of 899\n",
      "grabbing link /jobs/view/2632829080/, 196 of 899\n",
      "grabbing link /jobs/view/2666713394/, 197 of 899\n",
      "grabbing link /jobs/view/2655760321/, 198 of 899\n",
      "this link failed /jobs/view/2655760321/\n",
      "grabbing link /jobs/view/2601040270/, 199 of 899\n",
      "this link failed /jobs/view/2601040270/\n",
      "grabbing link /jobs/view/2661220781/, 200 of 899\n",
      "grabbing link /jobs/view/2646814209/, 201 of 899\n",
      "grabbing link /jobs/view/2658420854/, 202 of 899\n",
      "this link failed /jobs/view/2658420854/\n",
      "grabbing link /jobs/view/2660301021/, 203 of 899\n",
      "this link failed /jobs/view/2660301021/\n",
      "grabbing link /jobs/view/2659501025/, 204 of 899\n",
      "grabbing link /jobs/view/2616693581/, 205 of 899\n",
      "this link failed /jobs/view/2616693581/\n",
      "grabbing link /jobs/view/2600125874/, 206 of 899\n",
      "grabbing link /jobs/view/2642778749/, 207 of 899\n",
      "grabbing link /jobs/view/2642781539/, 208 of 899\n",
      "grabbing link /jobs/view/2641283505/, 209 of 899\n",
      "this link failed /jobs/view/2641283505/\n",
      "grabbing link /jobs/view/2637634832/, 210 of 899\n",
      "grabbing link /jobs/view/2656943801/, 211 of 899\n",
      "this link failed /jobs/view/2656943801/\n",
      "grabbing link /jobs/view/2653669409/, 212 of 899\n",
      "grabbing link /jobs/view/2583421013/, 213 of 899\n",
      "grabbing link /jobs/view/2620548225/, 214 of 899\n",
      "grabbing link /jobs/view/2656396279/, 215 of 899\n",
      "grabbing link /jobs/view/2664982344/, 216 of 899\n",
      "grabbing link /jobs/view/2654251226/, 217 of 899\n",
      "this link failed /jobs/view/2654251226/\n",
      "grabbing link /jobs/view/2649805831/, 218 of 899\n",
      "this link failed /jobs/view/2649805831/\n",
      "grabbing link /jobs/view/2626066710/, 219 of 899\n",
      "grabbing link /jobs/view/2645671813/, 220 of 899\n",
      "grabbing link /jobs/view/2651912259/, 221 of 899\n",
      "grabbing link /jobs/view/2661204303/, 222 of 899\n",
      "grabbing link /jobs/view/2651908787/, 223 of 899\n",
      "this link failed /jobs/view/2651908787/\n",
      "grabbing link /jobs/view/2616475124/, 224 of 899\n",
      "this link failed /jobs/view/2616475124/\n",
      "grabbing link /jobs/view/2600682270/, 225 of 899\n",
      "grabbing link /jobs/view/2640410452/, 226 of 899\n",
      "grabbing link /jobs/view/2661207756/, 227 of 899\n",
      "grabbing link /jobs/view/2608810047/, 228 of 899\n",
      "grabbing link /jobs/view/2666970912/, 229 of 899\n",
      "grabbing link /jobs/view/2522543760/, 230 of 899\n",
      "grabbing link /jobs/view/2666989227/, 231 of 899\n",
      "grabbing link /jobs/view/2643900511/, 232 of 899\n",
      "grabbing link /jobs/view/2647953120/, 233 of 899\n",
      "this link failed /jobs/view/2647953120/\n",
      "grabbing link /jobs/view/2637633870/, 234 of 899\n",
      "grabbing link /jobs/view/2657374650/, 235 of 899\n",
      "grabbing link /jobs/view/2660336253/, 236 of 899\n",
      "grabbing link /jobs/view/2616696523/, 237 of 899\n",
      "this link failed /jobs/view/2616696523/\n",
      "grabbing link /jobs/view/2629782641/, 238 of 899\n",
      "grabbing link /jobs/view/2661714914/, 239 of 899\n",
      "this link failed /jobs/view/2661714914/\n",
      "grabbing link /jobs/view/2628545316/, 240 of 899\n",
      "this link failed /jobs/view/2628545316/\n",
      "grabbing link /jobs/view/2637638239/, 241 of 899\n",
      "grabbing link /jobs/view/2628523596/, 242 of 899\n",
      "grabbing link /jobs/view/2600122929/, 243 of 899\n",
      "grabbing link /jobs/view/2664895230/, 244 of 899\n",
      "grabbing link /jobs/view/2577809203/, 245 of 899\n",
      "grabbing link /jobs/view/2604281637/, 246 of 899\n",
      "this link failed /jobs/view/2604281637/\n",
      "grabbing link /jobs/view/2642089624/, 247 of 899\n",
      "this link failed /jobs/view/2642089624/\n",
      "grabbing link /jobs/view/2600886884/, 248 of 899\n",
      "grabbing link /jobs/view/2637638241/, 249 of 899\n",
      "this link failed /jobs/view/2637638241/\n",
      "grabbing link /jobs/view/2631319223/, 250 of 899\n",
      "this link failed /jobs/view/2631319223/\n",
      "grabbing link /jobs/view/2519060100/, 251 of 899\n",
      "grabbing link /jobs/view/2656593744/, 252 of 899\n",
      "this link failed /jobs/view/2656593744/\n",
      "grabbing link /jobs/view/2642780624/, 253 of 899\n",
      "grabbing link /jobs/view/2642777834/, 254 of 899\n",
      "grabbing link /jobs/view/2666989185/, 255 of 899\n",
      "grabbing link /jobs/view/2623055792/, 256 of 899\n",
      "grabbing link /jobs/view/2660126101/, 257 of 899\n",
      "this link failed /jobs/view/2660126101/\n",
      "grabbing link /jobs/view/2654886971/, 258 of 899\n",
      "grabbing link /jobs/view/2659723856/, 259 of 899\n",
      "grabbing link /jobs/view/2661715765/, 260 of 899\n",
      "grabbing link /jobs/view/2658300191/, 261 of 899\n",
      "grabbing link /jobs/view/2641053593/, 262 of 899\n",
      "grabbing link /jobs/view/2645733665/, 263 of 899\n",
      "grabbing link /jobs/view/2607179068/, 264 of 899\n",
      "grabbing link /jobs/view/2661200789/, 265 of 899\n",
      "grabbing link /jobs/view/2657398228/, 266 of 899\n",
      "this link failed /jobs/view/2657398228/\n",
      "grabbing link /jobs/view/2659866261/, 267 of 899\n",
      "this link failed /jobs/view/2659866261/\n",
      "grabbing link /jobs/view/2600102998/, 268 of 899\n",
      "grabbing link /jobs/view/2649157820/, 269 of 899\n",
      "this link failed /jobs/view/2649157820/\n",
      "grabbing link /jobs/view/2661720237/, 270 of 899\n",
      "grabbing link /jobs/view/2660576128/, 271 of 899\n",
      "this link failed /jobs/view/2660576128/\n",
      "grabbing link /jobs/view/2651896147/, 272 of 899\n",
      "this link failed /jobs/view/2651896147/\n",
      "grabbing link /jobs/view/2615532289/, 273 of 899\n",
      "grabbing link /jobs/view/2630733761/, 274 of 899\n",
      "this link failed /jobs/view/2630733761/\n",
      "grabbing link /jobs/view/2650984181/, 275 of 899\n",
      "grabbing link /jobs/view/2617948061/, 276 of 899\n",
      "grabbing link /jobs/view/2650174068/, 277 of 899\n",
      "grabbing link /jobs/view/2660983683/, 278 of 899\n",
      "grabbing link /jobs/view/2656970687/, 279 of 899\n",
      "grabbing link /jobs/view/2661716619/, 280 of 899\n",
      "this link failed /jobs/view/2661716619/\n",
      "grabbing link /jobs/view/2656321670/, 281 of 899\n",
      "grabbing link /jobs/view/2648208072/, 282 of 899\n",
      "this link failed /jobs/view/2648208072/\n",
      "grabbing link /jobs/view/2660717719/, 283 of 899\n",
      "grabbing link /jobs/view/2600151926/, 284 of 899\n",
      "grabbing link /jobs/view/2438701808/, 285 of 899\n",
      "this link failed /jobs/view/2438701808/\n",
      "grabbing link /jobs/view/2665596709/, 286 of 899\n",
      "grabbing link /jobs/view/2651768193/, 287 of 899\n",
      "this link failed /jobs/view/2651768193/\n",
      "grabbing link /jobs/view/2661719259/, 288 of 899\n",
      "this link failed /jobs/view/2661719259/\n",
      "grabbing link /jobs/view/2601042359/, 289 of 899\n",
      "grabbing link /jobs/view/2627214064/, 290 of 899\n",
      "this link failed /jobs/view/2627214064/\n",
      "grabbing link /jobs/view/2654141546/, 291 of 899\n",
      "grabbing link /jobs/view/2659311106/, 292 of 899\n",
      "this link failed /jobs/view/2659311106/\n",
      "grabbing link /jobs/view/2659433768/, 293 of 899\n",
      "this link failed /jobs/view/2659433768/\n",
      "grabbing link /jobs/view/2600121129/, 294 of 899\n",
      "grabbing link /jobs/view/2657354824/, 295 of 899\n",
      "this link failed /jobs/view/2657354824/\n",
      "grabbing link /jobs/view/2652241985/, 296 of 899\n",
      "this link failed /jobs/view/2652241985/\n",
      "grabbing link /jobs/view/2591563261/, 297 of 899\n",
      "grabbing link /jobs/view/2634642690/, 298 of 899\n",
      "this link failed /jobs/view/2634642690/\n",
      "grabbing link /jobs/view/2657363691/, 299 of 899\n",
      "this link failed /jobs/view/2657363691/\n",
      "grabbing link /jobs/view/2619171185/, 300 of 899\n",
      "grabbing link /jobs/view/2656397396/, 301 of 899\n",
      "this link failed /jobs/view/2656397396/\n",
      "grabbing link /jobs/view/2643469286/, 302 of 899\n",
      "this link failed /jobs/view/2643469286/\n",
      "grabbing link /jobs/view/2633316709/, 303 of 899\n",
      "grabbing link /jobs/view/2667345283/, 304 of 899\n",
      "this link failed /jobs/view/2667345283/\n",
      "grabbing link /jobs/view/2629019427/, 305 of 899\n",
      "grabbing link /jobs/view/2615526982/, 306 of 899\n",
      "grabbing link /jobs/view/2406609624/, 307 of 899\n",
      "grabbing link /jobs/view/2666105934/, 308 of 899\n",
      "this link failed /jobs/view/2666105934/\n",
      "grabbing link /jobs/view/2656956290/, 309 of 899\n",
      "this link failed /jobs/view/2656956290/\n",
      "grabbing link /jobs/view/2664219825/, 310 of 899\n",
      "grabbing link /jobs/view/2656655239/, 311 of 899\n",
      "this link failed /jobs/view/2656655239/\n",
      "grabbing link /jobs/view/2653672443/, 312 of 899\n",
      "grabbing link /jobs/view/2614199156/, 313 of 899\n",
      "this link failed /jobs/view/2614199156/\n",
      "grabbing link /jobs/view/2616698399/, 314 of 899\n",
      "this link failed /jobs/view/2616698399/\n",
      "grabbing link /jobs/view/2646735308/, 315 of 899\n",
      "grabbing link /jobs/view/2651966156/, 316 of 899\n",
      "this link failed /jobs/view/2651966156/\n",
      "grabbing link /jobs/view/2522651066/, 317 of 899\n",
      "grabbing link /jobs/view/2616467431/, 318 of 899\n",
      "this link failed /jobs/view/2616467431/\n",
      "grabbing link /jobs/view/2629782287/, 319 of 899\n",
      "grabbing link /jobs/view/2650015928/, 320 of 899\n",
      "grabbing link /jobs/view/2658816414/, 321 of 899\n",
      "grabbing link /jobs/view/2654190567/, 322 of 899\n",
      "grabbing link /jobs/view/2661233240/, 323 of 899\n",
      "grabbing link /jobs/view/2479989653/, 324 of 899\n",
      "grabbing link /jobs/view/2265992835/, 325 of 899\n",
      "grabbing link /jobs/view/2666518058/, 326 of 899\n",
      "this link failed /jobs/view/2666518058/\n",
      "grabbing link /jobs/view/2662578388/, 327 of 899\n",
      "grabbing link /jobs/view/2653679332/, 328 of 899\n",
      "grabbing link /jobs/view/2604285358/, 329 of 899\n",
      "grabbing link /jobs/view/2649142930/, 330 of 899\n",
      "this link failed /jobs/view/2649142930/\n",
      "grabbing link /jobs/view/2601629355/, 331 of 899\n",
      "grabbing link /jobs/view/2591563258/, 332 of 899\n",
      "grabbing link /jobs/view/2579816550/, 333 of 899\n",
      "this link failed /jobs/view/2579816550/\n",
      "grabbing link /jobs/view/2656167250/, 334 of 899\n",
      "grabbing link /jobs/view/2641014137/, 335 of 899\n",
      "grabbing link /jobs/view/2658295549/, 336 of 899\n",
      "this link failed /jobs/view/2658295549/\n",
      "grabbing link /jobs/view/2558381847/, 337 of 899\n",
      "this link failed /jobs/view/2558381847/\n",
      "grabbing link /jobs/view/2556827406/, 338 of 899\n",
      "grabbing link /jobs/view/2628520506/, 339 of 899\n",
      "grabbing link /jobs/view/2640399624/, 340 of 899\n",
      "grabbing link /jobs/view/2661282664/, 341 of 899\n",
      "grabbing link /jobs/view/2654411000/, 342 of 899\n",
      "grabbing link /jobs/view/2656635056/, 343 of 899\n",
      "this link failed /jobs/view/2656635056/\n",
      "grabbing link /jobs/view/2654779475/, 344 of 899\n",
      "grabbing link /jobs/view/2658200325/, 345 of 899\n",
      "grabbing link /jobs/view/2667034680/, 346 of 899\n",
      "this link failed /jobs/view/2667034680/\n",
      "grabbing link /jobs/view/2633205277/, 347 of 899\n",
      "grabbing link /jobs/view/2666265357/, 348 of 899\n",
      "grabbing link /jobs/view/2600950590/, 349 of 899\n",
      "grabbing link /jobs/view/2651658751/, 350 of 899\n",
      "this link failed /jobs/view/2651658751/\n",
      "grabbing link /jobs/view/2656847673/, 351 of 899\n",
      "this link failed /jobs/view/2656847673/\n",
      "grabbing link /jobs/view/2616473580/, 352 of 899\n",
      "grabbing link /jobs/view/2629765637/, 353 of 899\n",
      "this link failed /jobs/view/2629765637/\n",
      "grabbing link /jobs/view/2645670935/, 354 of 899\n",
      "grabbing link /jobs/view/2651776010/, 355 of 899\n",
      "grabbing link /jobs/view/2659580476/, 356 of 899\n",
      "grabbing link /jobs/view/2654613013/, 357 of 899\n",
      "this link failed /jobs/view/2654613013/\n",
      "grabbing link /jobs/view/2658189801/, 358 of 899\n",
      "this link failed /jobs/view/2658189801/\n",
      "grabbing link /jobs/view/2625193532/, 359 of 899\n",
      "this link failed /jobs/view/2625193532/\n",
      "grabbing link /jobs/view/2635695005/, 360 of 899\n",
      "grabbing link /jobs/view/2655788354/, 361 of 899\n",
      "grabbing link /jobs/view/2658795125/, 362 of 899\n",
      "grabbing link /jobs/view/2658396122/, 363 of 899\n",
      "this link failed /jobs/view/2658396122/\n",
      "grabbing link /jobs/view/2654004414/, 364 of 899\n",
      "grabbing link /jobs/view/2652127450/, 365 of 899\n",
      "this link failed /jobs/view/2652127450/\n",
      "grabbing link /jobs/view/2656652573/, 366 of 899\n",
      "this link failed /jobs/view/2656652573/\n",
      "grabbing link /jobs/view/2598008812/, 367 of 899\n",
      "grabbing link /jobs/view/2650042601/, 368 of 899\n",
      "this link failed /jobs/view/2650042601/\n",
      "grabbing link /jobs/view/2653473908/, 369 of 899\n",
      "this link failed /jobs/view/2653473908/\n",
      "grabbing link /jobs/view/2616474211/, 370 of 899\n",
      "grabbing link /jobs/view/2650128419/, 371 of 899\n",
      "this link failed /jobs/view/2650128419/\n",
      "grabbing link /jobs/view/2650771795/, 372 of 899\n",
      "grabbing link /jobs/view/2651891958/, 373 of 899\n",
      "this link failed /jobs/view/2651891958/\n",
      "grabbing link /jobs/view/2626640127/, 374 of 899\n",
      "grabbing link /jobs/view/2659236311/, 375 of 899\n",
      "grabbing link /jobs/view/2618715816/, 376 of 899\n",
      "grabbing link /jobs/view/2656347062/, 377 of 899\n",
      "this link failed /jobs/view/2656347062/\n",
      "grabbing link /jobs/view/2628773690/, 378 of 899\n",
      "this link failed /jobs/view/2628773690/\n",
      "grabbing link /jobs/view/2665616284/, 379 of 899\n",
      "grabbing link /jobs/view/2642009620/, 380 of 899\n",
      "grabbing link /jobs/view/2641000593/, 381 of 899\n",
      "grabbing link /jobs/view/2619814102/, 382 of 899\n",
      "grabbing link /jobs/view/2649750099/, 383 of 899\n",
      "grabbing link /jobs/view/2658089280/, 384 of 899\n",
      "this link failed /jobs/view/2658089280/\n",
      "grabbing link /jobs/view/2626630489/, 385 of 899\n",
      "this link failed /jobs/view/2626630489/\n",
      "grabbing link /jobs/view/2619812113/, 386 of 899\n",
      "grabbing link /jobs/view/2625169692/, 387 of 899\n",
      "grabbing link /jobs/view/2657507282/, 388 of 899\n",
      "grabbing link /jobs/view/2604284359/, 389 of 899\n",
      "grabbing link /jobs/view/2612633133/, 390 of 899\n",
      "grabbing link /jobs/view/2629887180/, 391 of 899\n",
      "this link failed /jobs/view/2629887180/\n",
      "grabbing link /jobs/view/2660576086/, 392 of 899\n",
      "this link failed /jobs/view/2660576086/\n",
      "grabbing link /jobs/view/2626657251/, 393 of 899\n",
      "grabbing link /jobs/view/2607198911/, 394 of 899\n",
      "this link failed /jobs/view/2607198911/\n",
      "grabbing link /jobs/view/2651893748/, 395 of 899\n",
      "this link failed /jobs/view/2651893748/\n",
      "grabbing link /jobs/view/2664933366/, 396 of 899\n",
      "grabbing link /jobs/view/2590988353/, 397 of 899\n",
      "this link failed /jobs/view/2590988353/\n",
      "grabbing link /jobs/view/2655829206/, 398 of 899\n",
      "grabbing link /jobs/view/2661248994/, 399 of 899\n",
      "grabbing link /jobs/view/2658825835/, 400 of 899\n",
      "this link failed /jobs/view/2658825835/\n",
      "grabbing link /jobs/view/2657393880/, 401 of 899\n",
      "grabbing link /jobs/view/2654370117/, 402 of 899\n",
      "this link failed /jobs/view/2654370117/\n",
      "grabbing link /jobs/view/2658211338/, 403 of 899\n",
      "this link failed /jobs/view/2658211338/\n",
      "grabbing link /jobs/view/2615564189/, 404 of 899\n",
      "grabbing link /jobs/view/2571890771/, 405 of 899\n",
      "this link failed /jobs/view/2571890771/\n",
      "grabbing link /jobs/view/2643473225/, 406 of 899\n",
      "grabbing link /jobs/view/2637689082/, 407 of 899\n",
      "grabbing link /jobs/view/2659252158/, 408 of 899\n",
      "this link failed /jobs/view/2659252158/\n",
      "grabbing link /jobs/view/2587531317/, 409 of 899\n",
      "grabbing link /jobs/view/2636787643/, 410 of 899\n",
      "this link failed /jobs/view/2636787643/\n",
      "grabbing link /jobs/view/2650602017/, 411 of 899\n",
      "this link failed /jobs/view/2650602017/\n",
      "grabbing link /jobs/view/2642732687/, 412 of 899\n",
      "grabbing link /jobs/view/2632824859/, 413 of 899\n",
      "grabbing link /jobs/view/2658805781/, 414 of 899\n",
      "this link failed /jobs/view/2658805781/\n",
      "grabbing link /jobs/view/2641053591/, 415 of 899\n",
      "grabbing link /jobs/view/2658251126/, 416 of 899\n",
      "grabbing link /jobs/view/2590988334/, 417 of 899\n",
      "grabbing link /jobs/view/2656320646/, 418 of 899\n",
      "this link failed /jobs/view/2656320646/\n",
      "grabbing link /jobs/view/2633430577/, 419 of 899\n",
      "this link failed /jobs/view/2633430577/\n",
      "grabbing link /jobs/view/2645905416/, 420 of 899\n",
      "this link failed /jobs/view/2645905416/\n",
      "grabbing link /jobs/view/2630107750/, 421 of 899\n",
      "grabbing link /jobs/view/2660058777/, 422 of 899\n",
      "this link failed /jobs/view/2660058777/\n",
      "grabbing link /jobs/view/2638422344/, 423 of 899\n",
      "this link failed /jobs/view/2638422344/\n",
      "grabbing link /jobs/view/2650928906/, 424 of 899\n",
      "grabbing link /jobs/view/2652779631/, 425 of 899\n",
      "this link failed /jobs/view/2652779631/\n",
      "grabbing link /jobs/view/2666511927/, 426 of 899\n",
      "this link failed /jobs/view/2666511927/\n",
      "grabbing link /jobs/view/2641057241/, 427 of 899\n",
      "grabbing link /jobs/view/2614793364/, 428 of 899\n",
      "this link failed /jobs/view/2614793364/\n",
      "grabbing link /jobs/view/2621697236/, 429 of 899\n",
      "grabbing link /jobs/view/2625828644/, 430 of 899\n",
      "this link failed /jobs/view/2625828644/\n",
      "grabbing link /jobs/view/2655035490/, 431 of 899\n",
      "this link failed /jobs/view/2655035490/\n",
      "grabbing link /jobs/view/2656336161/, 432 of 899\n",
      "grabbing link /jobs/view/2664225285/, 433 of 899\n",
      "grabbing link /jobs/view/2646286650/, 434 of 899\n",
      "grabbing link /jobs/view/2621694436/, 435 of 899\n",
      "grabbing link /jobs/view/2580861163/, 436 of 899\n",
      "this link failed /jobs/view/2580861163/\n",
      "grabbing link /jobs/view/2657938869/, 437 of 899\n",
      "grabbing link /jobs/view/2665382803/, 438 of 899\n",
      "this link failed /jobs/view/2665382803/\n",
      "grabbing link /jobs/view/2604885852/, 439 of 899\n",
      "grabbing link /jobs/view/2601239259/, 440 of 899\n",
      "this link failed /jobs/view/2601239259/\n",
      "grabbing link /jobs/view/2628904886/, 441 of 899\n",
      "this link failed /jobs/view/2628904886/\n",
      "grabbing link /jobs/view/2630724591/, 442 of 899\n",
      "grabbing link /jobs/view/2658063833/, 443 of 899\n",
      "this link failed /jobs/view/2658063833/\n",
      "grabbing link /jobs/view/2641057243/, 444 of 899\n",
      "grabbing link /jobs/view/2640347410/, 445 of 899\n",
      "grabbing link /jobs/view/2647341732/, 446 of 899\n",
      "grabbing link /jobs/view/2649134983/, 447 of 899\n",
      "grabbing link /jobs/view/2626627243/, 448 of 899\n",
      "grabbing link /jobs/view/2658875091/, 449 of 899\n",
      "this link failed /jobs/view/2658875091/\n",
      "grabbing link /jobs/view/2650131901/, 450 of 899\n",
      "this link failed /jobs/view/2650131901/\n",
      "grabbing link /jobs/view/2438706198/, 451 of 899\n",
      "grabbing link /jobs/view/2658209342/, 452 of 899\n",
      "this link failed /jobs/view/2658209342/\n",
      "grabbing link /jobs/view/2653097093/, 453 of 899\n",
      "grabbing link /jobs/view/2640396980/, 454 of 899\n",
      "grabbing link /jobs/view/2632000248/, 455 of 899\n",
      "grabbing link /jobs/view/2650782753/, 456 of 899\n",
      "this link failed /jobs/view/2650782753/\n",
      "grabbing link /jobs/view/2659563428/, 457 of 899\n",
      "grabbing link /jobs/view/2645670165/, 458 of 899\n",
      "grabbing link /jobs/view/2618791806/, 459 of 899\n",
      "grabbing link /jobs/view/2647530722/, 460 of 899\n",
      "grabbing link /jobs/view/2655475913/, 461 of 899\n",
      "grabbing link /jobs/view/2661240787/, 462 of 899\n",
      "grabbing link /jobs/view/2642096285/, 463 of 899\n",
      "grabbing link /jobs/view/2664564812/, 464 of 899\n",
      "this link failed /jobs/view/2664564812/\n",
      "grabbing link /jobs/view/2663907667/, 465 of 899\n",
      "this link failed /jobs/view/2663907667/\n",
      "grabbing link /jobs/view/2618751551/, 466 of 899\n",
      "grabbing link /jobs/view/2658812332/, 467 of 899\n",
      "this link failed /jobs/view/2658812332/\n",
      "grabbing link /jobs/view/2647522401/, 468 of 899\n",
      "grabbing link /jobs/view/2658297127/, 469 of 899\n",
      "this link failed /jobs/view/2658297127/\n",
      "grabbing link /jobs/view/2660704268/, 470 of 899\n",
      "grabbing link /jobs/view/2653335852/, 471 of 899\n",
      "grabbing link /jobs/view/2667035574/, 472 of 899\n",
      "grabbing link /jobs/view/2660321590/, 473 of 899\n",
      "grabbing link /jobs/view/2604943170/, 474 of 899\n",
      "this link failed /jobs/view/2604943170/\n",
      "grabbing link /jobs/view/2583327379/, 475 of 899\n",
      "grabbing link /jobs/view/2662377329/, 476 of 899\n",
      "grabbing link /jobs/view/2658970269/, 477 of 899\n",
      "grabbing link /jobs/view/2654403853/, 478 of 899\n",
      "grabbing link /jobs/view/2659513896/, 479 of 899\n",
      "grabbing link /jobs/view/2616508604/, 480 of 899\n",
      "grabbing link /jobs/view/2488357390/, 481 of 899\n",
      "this link failed /jobs/view/2488357390/\n",
      "grabbing link /jobs/view/2624556362/, 482 of 899\n",
      "this link failed /jobs/view/2624556362/\n",
      "grabbing link /jobs/view/2628512060/, 483 of 899\n",
      "grabbing link /jobs/view/2639212685/, 484 of 899\n",
      "grabbing link /jobs/view/2628520507/, 485 of 899\n",
      "this link failed /jobs/view/2628520507/\n",
      "grabbing link /jobs/view/2633316708/, 486 of 899\n",
      "this link failed /jobs/view/2633316708/\n",
      "grabbing link /jobs/view/2616512248/, 487 of 899\n",
      "this link failed /jobs/view/2616512248/\n",
      "grabbing link /jobs/view/2634655427/, 488 of 899\n",
      "grabbing link /jobs/view/2652736902/, 489 of 899\n",
      "this link failed /jobs/view/2652736902/\n",
      "grabbing link /jobs/view/2659081630/, 490 of 899\n",
      "grabbing link /jobs/view/2635950493/, 491 of 899\n",
      "this link failed /jobs/view/2635950493/\n",
      "grabbing link /jobs/view/2658295591/, 492 of 899\n",
      "this link failed /jobs/view/2658295591/\n",
      "grabbing link /jobs/view/2649150398/, 493 of 899\n",
      "grabbing link /jobs/view/2633745284/, 494 of 899\n",
      "grabbing link /jobs/view/2658197254/, 495 of 899\n",
      "this link failed /jobs/view/2658197254/\n",
      "grabbing link /jobs/view/2650049895/, 496 of 899\n",
      "grabbing link /jobs/view/2627255926/, 497 of 899\n",
      "this link failed /jobs/view/2627255926/\n",
      "grabbing link /jobs/view/2625189850/, 498 of 899\n",
      "grabbing link /jobs/view/2599197959/, 499 of 899\n",
      "this link failed /jobs/view/2599197959/\n",
      "grabbing link /jobs/view/2637688233/, 500 of 899\n",
      "grabbing link /jobs/view/2665756138/, 501 of 899\n",
      "this link failed /jobs/view/2665756138/\n",
      "grabbing link /jobs/view/2633216061/, 502 of 899\n",
      "this link failed /jobs/view/2633216061/\n",
      "grabbing link /jobs/view/2652787353/, 503 of 899\n",
      "grabbing link /jobs/view/2637603025/, 504 of 899\n",
      "grabbing link /jobs/view/2571213707/, 505 of 899\n",
      "grabbing link /jobs/view/2643679732/, 506 of 899\n",
      "grabbing link /jobs/view/2628505750/, 507 of 899\n",
      "grabbing link /jobs/view/2637689080/, 508 of 899\n",
      "grabbing link /jobs/view/2639200746/, 509 of 899\n",
      "grabbing link /jobs/view/2637602027/, 510 of 899\n",
      "grabbing link /jobs/view/2646809981/, 511 of 899\n",
      "grabbing link /jobs/view/2654402797/, 512 of 899\n",
      "grabbing link /jobs/view/2652247144/, 513 of 899\n",
      "grabbing link /jobs/view/2645675185/, 514 of 899\n",
      "grabbing link /jobs/view/2667030969/, 515 of 899\n",
      "grabbing link /jobs/view/2655378894/, 516 of 899\n",
      "grabbing link /jobs/view/2633710834/, 517 of 899\n",
      "grabbing link /jobs/view/2592662480/, 518 of 899\n",
      "grabbing link /jobs/view/2667358779/, 519 of 899\n",
      "grabbing link /jobs/view/2625198586/, 520 of 899\n",
      "grabbing link /jobs/view/2615009163/, 521 of 899\n",
      "grabbing link /jobs/view/2666286507/, 522 of 899\n",
      "grabbing link /jobs/view/2633204330/, 523 of 899\n",
      "grabbing link /jobs/view/2640563902/, 524 of 899\n",
      "grabbing link /jobs/view/2660782585/, 525 of 899\n",
      "grabbing link /jobs/view/2645858266/, 526 of 899\n",
      "grabbing link /jobs/view/2601109748/, 527 of 899\n",
      "grabbing link /jobs/view/2637685682/, 528 of 899\n",
      "grabbing link /jobs/view/2595657729/, 529 of 899\n",
      "grabbing link /jobs/view/2633953053/, 530 of 899\n",
      "grabbing link /jobs/view/2486220634/, 531 of 899\n",
      "grabbing link /jobs/view/2628902040/, 532 of 899\n",
      "grabbing link /jobs/view/2649181505/, 533 of 899\n",
      "grabbing link /jobs/view/2619870217/, 534 of 899\n",
      "grabbing link /jobs/view/2645671827/, 535 of 899\n",
      "grabbing link /jobs/view/2625111059/, 536 of 899\n",
      "grabbing link /jobs/view/2636813604/, 537 of 899\n",
      "grabbing link /jobs/view/2582657614/, 538 of 899\n",
      "grabbing link /jobs/view/2652394215/, 539 of 899\n",
      "grabbing link /jobs/view/2663910664/, 540 of 899\n",
      "grabbing link /jobs/view/2655365527/, 541 of 899\n",
      "this link failed /jobs/view/2655365527/\n",
      "grabbing link /jobs/view/2623822955/, 542 of 899\n",
      "grabbing link /jobs/view/2652786425/, 543 of 899\n",
      "grabbing link /jobs/view/2557483056/, 544 of 899\n",
      "grabbing link /jobs/view/2657715046/, 545 of 899\n",
      "this link failed /jobs/view/2657715046/\n",
      "grabbing link /jobs/view/2658869810/, 546 of 899\n",
      "this link failed /jobs/view/2658869810/\n",
      "grabbing link /jobs/view/2630108455/, 547 of 899\n",
      "grabbing link /jobs/view/2556825924/, 548 of 899\n",
      "grabbing link /jobs/view/2656774432/, 549 of 899\n",
      "grabbing link /jobs/view/2635828046/, 550 of 899\n",
      "this link failed /jobs/view/2635828046/\n",
      "grabbing link /jobs/view/2630723787/, 551 of 899\n",
      "this link failed /jobs/view/2630723787/\n",
      "grabbing link /jobs/view/2638622710/, 552 of 899\n",
      "grabbing link /jobs/view/2661917820/, 553 of 899\n",
      "this link failed /jobs/view/2661917820/\n",
      "grabbing link /jobs/view/2652753313/, 554 of 899\n",
      "this link failed /jobs/view/2652753313/\n",
      "grabbing link /jobs/view/2661391047/, 555 of 899\n",
      "grabbing link /jobs/view/2630100803/, 556 of 899\n",
      "grabbing link /jobs/view/2660320739/, 557 of 899\n",
      "grabbing link /jobs/view/2665611787/, 558 of 899\n",
      "grabbing link /jobs/view/2659697333/, 559 of 899\n",
      "grabbing link /jobs/view/2656797031/, 560 of 899\n",
      "this link failed /jobs/view/2656797031/\n",
      "grabbing link /jobs/view/2636899587/, 561 of 899\n",
      "grabbing link /jobs/view/2580740910/, 562 of 899\n",
      "this link failed /jobs/view/2580740910/\n",
      "grabbing link /jobs/view/2666499741/, 563 of 899\n",
      "grabbing link /jobs/view/2660108770/, 564 of 899\n",
      "this link failed /jobs/view/2660108770/\n",
      "grabbing link /jobs/view/2631320821/, 565 of 899\n",
      "this link failed /jobs/view/2631320821/\n",
      "grabbing link /jobs/view/2637601314/, 566 of 899\n",
      "grabbing link /jobs/view/2636630025/, 567 of 899\n",
      "grabbing link /jobs/view/2660113313/, 568 of 899\n",
      "grabbing link /jobs/view/2652066815/, 569 of 899\n",
      "this link failed /jobs/view/2652066815/\n",
      "grabbing link /jobs/view/2535408585/, 570 of 899\n",
      "this link failed /jobs/view/2535408585/\n",
      "grabbing link /jobs/view/2637602089/, 571 of 899\n",
      "this link failed /jobs/view/2637602089/\n",
      "grabbing link /jobs/view/2567520265/, 572 of 899\n",
      "this link failed /jobs/view/2567520265/\n",
      "grabbing link /jobs/view/2660325271/, 573 of 899\n",
      "grabbing link /jobs/view/2660326130/, 574 of 899\n",
      "grabbing link /jobs/view/2506772795/, 575 of 899\n",
      "grabbing link /jobs/view/2522297078/, 576 of 899\n",
      "this link failed /jobs/view/2522297078/\n",
      "grabbing link /jobs/view/2657378274/, 577 of 899\n",
      "grabbing link /jobs/view/2633710855/, 578 of 899\n",
      "grabbing link /jobs/view/2656344166/, 579 of 899\n",
      "this link failed /jobs/view/2656344166/\n",
      "grabbing link /jobs/view/2642008754/, 580 of 899\n",
      "grabbing link /jobs/view/2640332710/, 581 of 899\n",
      "grabbing link /jobs/view/2638160949/, 582 of 899\n",
      "grabbing link /jobs/view/2580846425/, 583 of 899\n",
      "grabbing link /jobs/view/2579852025/, 584 of 899\n",
      "this link failed /jobs/view/2579852025/\n",
      "grabbing link /jobs/view/2630766131/, 585 of 899\n",
      "grabbing link /jobs/view/2659917325/, 586 of 899\n",
      "grabbing link /jobs/view/2649745724/, 587 of 899\n",
      "grabbing link /jobs/view/2661258924/, 588 of 899\n",
      "this link failed /jobs/view/2661258924/\n",
      "grabbing link /jobs/view/2617221177/, 589 of 899\n",
      "grabbing link /jobs/view/2654914146/, 590 of 899\n",
      "grabbing link /jobs/view/2647557704/, 591 of 899\n",
      "grabbing link /jobs/view/2603620389/, 592 of 899\n",
      "this link failed /jobs/view/2603620389/\n",
      "grabbing link /jobs/view/2625801879/, 593 of 899\n",
      "grabbing link /jobs/view/2646700688/, 594 of 899\n",
      "grabbing link /jobs/view/2616512148/, 595 of 899\n",
      "grabbing link /jobs/view/2622290816/, 596 of 899\n",
      "this link failed /jobs/view/2622290816/\n",
      "grabbing link /jobs/view/2627632915/, 597 of 899\n",
      "grabbing link /jobs/view/2640146628/, 598 of 899\n",
      "grabbing link /jobs/view/2648515852/, 599 of 899\n",
      "this link failed /jobs/view/2648515852/\n",
      "grabbing link /jobs/view/2660903646/, 600 of 899\n",
      "grabbing link /jobs/view/2633141780/, 601 of 899\n",
      "grabbing link /jobs/view/2657044734/, 602 of 899\n",
      "grabbing link /jobs/view/2633947802/, 603 of 899\n",
      "grabbing link /jobs/view/2556825926/, 604 of 899\n",
      "this link failed /jobs/view/2556825926/\n",
      "grabbing link /jobs/view/2616513000/, 605 of 899\n",
      "grabbing link /jobs/view/2657324558/, 606 of 899\n",
      "grabbing link /jobs/view/2630103646/, 607 of 899\n",
      "grabbing link /jobs/view/2637148331/, 608 of 899\n",
      "this link failed /jobs/view/2637148331/\n",
      "grabbing link /jobs/view/2661260729/, 609 of 899\n",
      "this link failed /jobs/view/2661260729/\n",
      "grabbing link /jobs/view/2658966854/, 610 of 899\n",
      "grabbing link /jobs/view/2581268870/, 611 of 899\n",
      "this link failed /jobs/view/2581268870/\n",
      "grabbing link /jobs/view/2581261767/, 612 of 899\n",
      "this link failed /jobs/view/2581261767/\n",
      "grabbing link /jobs/view/2659911992/, 613 of 899\n",
      "grabbing link /jobs/view/2620553932/, 614 of 899\n",
      "grabbing link /jobs/view/2651966868/, 615 of 899\n",
      "grabbing link /jobs/view/2650918183/, 616 of 899\n",
      "this link failed /jobs/view/2650918183/\n",
      "grabbing link /jobs/view/2619864848/, 617 of 899\n",
      "this link failed /jobs/view/2619864848/\n",
      "grabbing link /jobs/view/2658215711/, 618 of 899\n",
      "grabbing link /jobs/view/2646501151/, 619 of 899\n",
      "grabbing link /jobs/view/2645651184/, 620 of 899\n",
      "grabbing link /jobs/view/2647549380/, 621 of 899\n",
      "grabbing link /jobs/view/2663908733/, 622 of 899\n",
      "grabbing link /jobs/view/2616508612/, 623 of 899\n",
      "grabbing link /jobs/view/2658869811/, 624 of 899\n",
      "grabbing link /jobs/view/2649144940/, 625 of 899\n",
      "grabbing link /jobs/view/2616507800/, 626 of 899\n",
      "grabbing link /jobs/view/2646343105/, 627 of 899\n",
      "grabbing link /jobs/view/2592516438/, 628 of 899\n",
      "grabbing link /jobs/view/2661258922/, 629 of 899\n",
      "grabbing link /jobs/view/2634215868/, 630 of 899\n",
      "grabbing link /jobs/view/2624903494/, 631 of 899\n",
      "grabbing link /jobs/view/2619870215/, 632 of 899\n",
      "grabbing link /jobs/view/2646310409/, 633 of 899\n",
      "grabbing link /jobs/view/2666515444/, 634 of 899\n",
      "grabbing link /jobs/view/2643923989/, 635 of 899\n",
      "grabbing link /jobs/view/2658214190/, 636 of 899\n",
      "grabbing link /jobs/view/2472224623/, 637 of 899\n",
      "grabbing link /jobs/view/2649173442/, 638 of 899\n",
      "grabbing link /jobs/view/2658868979/, 639 of 899\n",
      "grabbing link /jobs/view/2593988315/, 640 of 899\n",
      "grabbing link /jobs/view/2548682320/, 641 of 899\n",
      "grabbing link /jobs/view/2633714504/, 642 of 899\n",
      "grabbing link /jobs/view/2600654054/, 643 of 899\n",
      "grabbing link /jobs/view/2597860195/, 644 of 899\n",
      "grabbing link /jobs/view/2659435707/, 645 of 899\n",
      "grabbing link /jobs/view/2642013399/, 646 of 899\n",
      "grabbing link /jobs/view/2660775186/, 647 of 899\n",
      "grabbing link /jobs/view/2661296532/, 648 of 899\n",
      "grabbing link /jobs/view/2656859862/, 649 of 899\n",
      "this link failed /jobs/view/2656859862/\n",
      "grabbing link /jobs/view/2643972382/, 650 of 899\n",
      "this link failed /jobs/view/2643972382/\n",
      "grabbing link /jobs/view/2618756017/, 651 of 899\n",
      "grabbing link /jobs/view/2666496914/, 652 of 899\n",
      "this link failed /jobs/view/2666496914/\n",
      "grabbing link /jobs/view/2486220651/, 653 of 899\n",
      "this link failed /jobs/view/2486220651/\n",
      "grabbing link /jobs/view/2643682389/, 654 of 899\n",
      "grabbing link /jobs/view/2658214426/, 655 of 899\n",
      "this link failed /jobs/view/2658214426/\n",
      "grabbing link /jobs/view/2582613069/, 656 of 899\n",
      "grabbing link /jobs/view/2650785207/, 657 of 899\n",
      "grabbing link /jobs/view/2657392025/, 658 of 899\n",
      "grabbing link /jobs/view/2654398035/, 659 of 899\n",
      "grabbing link /jobs/view/2625606920/, 660 of 899\n",
      "grabbing link /jobs/view/2660719650/, 661 of 899\n",
      "grabbing link /jobs/view/2656183338/, 662 of 899\n",
      "grabbing link /jobs/view/2636815319/, 663 of 899\n",
      "grabbing link /jobs/view/2639380260/, 664 of 899\n",
      "grabbing link /jobs/view/2598024329/, 665 of 899\n",
      "grabbing link /jobs/view/2656850001/, 666 of 899\n",
      "grabbing link /jobs/view/2658961458/, 667 of 899\n",
      "grabbing link /jobs/view/2656848269/, 668 of 899\n",
      "grabbing link /jobs/view/2646777680/, 669 of 899\n",
      "grabbing link /jobs/view/2658227025/, 670 of 899\n",
      "grabbing link /jobs/view/2654576391/, 671 of 899\n",
      "grabbing link /jobs/view/2552490103/, 672 of 899\n",
      "grabbing link /jobs/view/2659511685/, 673 of 899\n",
      "grabbing link /jobs/view/2638506391/, 674 of 899\n",
      "grabbing link /jobs/view/2657052027/, 675 of 899\n",
      "grabbing link /jobs/view/2645134014/, 676 of 899\n",
      "grabbing link /jobs/view/2547090924/, 677 of 899\n",
      "grabbing link /jobs/view/2650967255/, 678 of 899\n",
      "grabbing link /jobs/view/2658921632/, 679 of 899\n",
      "grabbing link /jobs/view/2635827143/, 680 of 899\n",
      "grabbing link /jobs/view/2631240683/, 681 of 899\n",
      "grabbing link /jobs/view/2619864259/, 682 of 899\n",
      "grabbing link /jobs/view/2663176748/, 683 of 899\n",
      "grabbing link /jobs/view/2577849105/, 684 of 899\n",
      "grabbing link /jobs/view/2636819595/, 685 of 899\n",
      "grabbing link /jobs/view/2632824856/, 686 of 899\n",
      "grabbing link /jobs/view/2650539338/, 687 of 899\n",
      "grabbing link /jobs/view/2649160209/, 688 of 899\n",
      "grabbing link /jobs/view/2660773911/, 689 of 899\n",
      "this link failed /jobs/view/2660773911/\n",
      "grabbing link /jobs/view/2658673626/, 690 of 899\n",
      "grabbing link /jobs/view/2649116730/, 691 of 899\n",
      "grabbing link /jobs/view/2655888426/, 692 of 899\n",
      "grabbing link /jobs/view/2658193201/, 693 of 899\n",
      "this link failed /jobs/view/2658193201/\n",
      "grabbing link /jobs/view/2654139926/, 694 of 899\n",
      "grabbing link /jobs/view/2629787352/, 695 of 899\n",
      "grabbing link /jobs/view/2663912532/, 696 of 899\n",
      "this link failed /jobs/view/2663912532/\n",
      "grabbing link /jobs/view/2660445020/, 697 of 899\n",
      "grabbing link /jobs/view/2659250330/, 698 of 899\n",
      "this link failed /jobs/view/2659250330/\n",
      "grabbing link /jobs/view/2654575439/, 699 of 899\n",
      "this link failed /jobs/view/2654575439/\n",
      "grabbing link /jobs/view/2625874563/, 700 of 899\n",
      "this link failed /jobs/view/2625874563/\n",
      "grabbing link /jobs/view/2652061934/, 701 of 899\n",
      "this link failed /jobs/view/2652061934/\n",
      "grabbing link /jobs/view/2659508573/, 702 of 899\n",
      "grabbing link /jobs/view/2645643899/, 703 of 899\n",
      "grabbing link /jobs/view/2654275788/, 704 of 899\n",
      "grabbing link /jobs/view/2650493744/, 705 of 899\n",
      "grabbing link /jobs/view/2649136825/, 706 of 899\n",
      "grabbing link /jobs/view/2652112511/, 707 of 899\n",
      "grabbing link /jobs/view/2533522962/, 708 of 899\n",
      "grabbing link /jobs/view/2657087936/, 709 of 899\n",
      "grabbing link /jobs/view/2658879830/, 710 of 899\n",
      "grabbing link /jobs/view/2637136012/, 711 of 899\n",
      "this link failed /jobs/view/2637136012/\n",
      "grabbing link /jobs/view/2660760902/, 712 of 899\n",
      "this link failed /jobs/view/2660760902/\n",
      "grabbing link /jobs/view/2619869448/, 713 of 899\n",
      "grabbing link /jobs/view/2658283529/, 714 of 899\n",
      "grabbing link /jobs/view/2654580222/, 715 of 899\n",
      "grabbing link /jobs/view/2626629855/, 716 of 899\n",
      "grabbing link /jobs/view/2659082786/, 717 of 899\n",
      "grabbing link /jobs/view/2656969683/, 718 of 899\n",
      "grabbing link /jobs/view/2640361232/, 719 of 899\n",
      "this link failed /jobs/view/2640361232/\n",
      "grabbing link /jobs/view/2621809732/, 720 of 899\n",
      "grabbing link /jobs/view/2653006690/, 721 of 899\n",
      "this link failed /jobs/view/2653006690/\n",
      "grabbing link /jobs/view/2485547106/, 722 of 899\n",
      "grabbing link /jobs/view/2638888005/, 723 of 899\n",
      "grabbing link /jobs/view/2620502701/, 724 of 899\n",
      "grabbing link /jobs/view/2630112080/, 725 of 899\n",
      "grabbing link /jobs/view/2652760387/, 726 of 899\n",
      "grabbing link /jobs/view/2630795324/, 727 of 899\n",
      "grabbing link /jobs/view/2626680252/, 728 of 899\n",
      "this link failed /jobs/view/2626680252/\n",
      "grabbing link /jobs/view/2653782045/, 729 of 899\n",
      "this link failed /jobs/view/2653782045/\n",
      "grabbing link /jobs/view/2637605250/, 730 of 899\n",
      "grabbing link /jobs/view/2642748740/, 731 of 899\n",
      "this link failed /jobs/view/2642748740/\n",
      "grabbing link /jobs/view/2666161120/, 732 of 899\n",
      "grabbing link /jobs/view/2629937647/, 733 of 899\n",
      "this link failed /jobs/view/2629937647/\n",
      "grabbing link /jobs/view/2658225322/, 734 of 899\n",
      "grabbing link /jobs/view/2630130084/, 735 of 899\n",
      "grabbing link /jobs/view/2659573393/, 736 of 899\n",
      "grabbing link /jobs/view/2659573396/, 737 of 899\n",
      "this link failed /jobs/view/2659573396/\n",
      "grabbing link /jobs/view/2643972653/, 738 of 899\n",
      "grabbing link /jobs/view/2454029283/, 739 of 899\n",
      "this link failed /jobs/view/2454029283/\n",
      "grabbing link /jobs/view/2642499588/, 740 of 899\n",
      "this link failed /jobs/view/2642499588/\n",
      "grabbing link /jobs/view/2650439098/, 741 of 899\n",
      "this link failed /jobs/view/2650439098/\n",
      "grabbing link /jobs/view/2626626459/, 742 of 899\n",
      "grabbing link /jobs/view/2629788151/, 743 of 899\n",
      "this link failed /jobs/view/2629788151/\n",
      "grabbing link /jobs/view/2642089714/, 744 of 899\n",
      "grabbing link /jobs/view/2659919063/, 745 of 899\n",
      "grabbing link /jobs/view/2658921635/, 746 of 899\n",
      "this link failed /jobs/view/2658921635/\n",
      "grabbing link /jobs/view/2654609382/, 747 of 899\n",
      "grabbing link /jobs/view/2653472639/, 748 of 899\n",
      "grabbing link /jobs/view/2655886968/, 749 of 899\n",
      "grabbing link /jobs/view/2469457203/, 750 of 899\n",
      "this link failed /jobs/view/2469457203/\n",
      "grabbing link /jobs/view/2654772083/, 751 of 899\n",
      "this link failed /jobs/view/2654772083/\n",
      "grabbing link /jobs/view/2652119898/, 752 of 899\n",
      "this link failed /jobs/view/2652119898/\n",
      "grabbing link /jobs/view/2657297063/, 753 of 899\n",
      "grabbing link /jobs/view/1718273657/, 754 of 899\n",
      "this link failed /jobs/view/1718273657/\n",
      "grabbing link /jobs/view/2666491189/, 755 of 899\n",
      "grabbing link /jobs/view/2659914662/, 756 of 899\n",
      "this link failed /jobs/view/2659914662/\n",
      "grabbing link /jobs/view/2650097234/, 757 of 899\n",
      "this link failed /jobs/view/2650097234/\n",
      "grabbing link /jobs/view/2651784734/, 758 of 899\n",
      "grabbing link /jobs/view/2639008266/, 759 of 899\n",
      "this link failed /jobs/view/2639008266/\n",
      "grabbing link /jobs/view/2657449531/, 760 of 899\n",
      "grabbing link /jobs/view/2540034567/, 761 of 899\n",
      "this link failed /jobs/view/2540034567/\n",
      "grabbing link /jobs/view/2642747829/, 762 of 899\n",
      "this link failed /jobs/view/2642747829/\n",
      "grabbing link /jobs/view/2649840594/, 763 of 899\n",
      "this link failed /jobs/view/2649840594/\n",
      "grabbing link /jobs/view/2649842551/, 764 of 899\n",
      "grabbing link /jobs/view/2339540364/, 765 of 899\n",
      "this link failed /jobs/view/2339540364/\n",
      "grabbing link /jobs/view/2640350369/, 766 of 899\n",
      "this link failed /jobs/view/2640350369/\n",
      "grabbing link /jobs/view/2654577358/, 767 of 899\n",
      "grabbing link /jobs/view/2640354392/, 768 of 899\n",
      "this link failed /jobs/view/2640354392/\n",
      "grabbing link /jobs/view/2645664727/, 769 of 899\n",
      "this link failed /jobs/view/2645664727/\n",
      "grabbing link /jobs/view/2656971427/, 770 of 899\n",
      "grabbing link /jobs/view/2661401959/, 771 of 899\n",
      "this link failed /jobs/view/2661401959/\n",
      "grabbing link /jobs/view/2639259832/, 772 of 899\n",
      "grabbing link /jobs/view/2660555403/, 773 of 899\n",
      "this link failed /jobs/view/2660555403/\n",
      "grabbing link /jobs/view/2650777026/, 774 of 899\n",
      "this link failed /jobs/view/2650777026/\n",
      "grabbing link /jobs/view/2631789091/, 775 of 899\n",
      "this link failed /jobs/view/2631789091/\n",
      "grabbing link /jobs/view/2634664986/, 776 of 899\n",
      "this link failed /jobs/view/2634664986/\n",
      "grabbing link /jobs/view/2656849974/, 777 of 899\n",
      "this link failed /jobs/view/2656849974/\n",
      "grabbing link /jobs/view/2654410022/, 778 of 899\n",
      "grabbing link /jobs/view/2645699994/, 779 of 899\n",
      "grabbing link /jobs/view/2639212269/, 780 of 899\n",
      "grabbing link /jobs/view/2633735563/, 781 of 899\n",
      "grabbing link /jobs/view/2641079584/, 782 of 899\n",
      "grabbing link /jobs/view/2654580213/, 783 of 899\n",
      "this link failed /jobs/view/2654580213/\n",
      "grabbing link /jobs/view/2634655726/, 784 of 899\n",
      "grabbing link /jobs/view/2656966918/, 785 of 899\n",
      "this link failed /jobs/view/2656966918/\n",
      "grabbing link /jobs/view/2667016940/, 786 of 899\n",
      "grabbing link /jobs/view/2595654754/, 787 of 899\n",
      "this link failed /jobs/view/2595654754/\n",
      "grabbing link /jobs/view/2626628381/, 788 of 899\n",
      "this link failed /jobs/view/2626628381/\n",
      "grabbing link /jobs/view/2647543503/, 789 of 899\n",
      "grabbing link /jobs/view/2666984773/, 790 of 899\n",
      "grabbing link /jobs/view/2655888042/, 791 of 899\n",
      "grabbing link /jobs/view/2642081560/, 792 of 899\n",
      "grabbing link /jobs/view/2636839362/, 793 of 899\n",
      "this link failed /jobs/view/2636839362/\n",
      "grabbing link /jobs/view/2644908943/, 794 of 899\n",
      "grabbing link /jobs/view/2648406582/, 795 of 899\n",
      "this link failed /jobs/view/2648406582/\n",
      "grabbing link /jobs/view/2625806509/, 796 of 899\n",
      "this link failed /jobs/view/2625806509/\n",
      "grabbing link /jobs/view/2625191426/, 797 of 899\n",
      "grabbing link /jobs/view/2649123792/, 798 of 899\n",
      "this link failed /jobs/view/2649123792/\n",
      "grabbing link /jobs/view/2519000044/, 799 of 899\n",
      "this link failed /jobs/view/2519000044/\n",
      "grabbing link /jobs/view/2460876721/, 800 of 899\n",
      "grabbing link /jobs/view/2649134638/, 801 of 899\n",
      "grabbing link /jobs/view/2658194175/, 802 of 899\n",
      "grabbing link /jobs/view/2636820685/, 803 of 899\n",
      "grabbing link /jobs/view/2654608390/, 804 of 899\n",
      "this link failed /jobs/view/2654608390/\n",
      "grabbing link /jobs/view/2654404932/, 805 of 899\n",
      "grabbing link /jobs/view/2630118293/, 806 of 899\n",
      "grabbing link /jobs/view/2640532604/, 807 of 899\n",
      "grabbing link /jobs/view/2636815826/, 808 of 899\n",
      "this link failed /jobs/view/2636815826/\n",
      "grabbing link /jobs/view/2642111473/, 809 of 899\n",
      "grabbing link /jobs/view/2460877781/, 810 of 899\n",
      "this link failed /jobs/view/2460877781/\n",
      "grabbing link /jobs/view/2640358043/, 811 of 899\n",
      "this link failed /jobs/view/2640358043/\n",
      "grabbing link /jobs/view/2641075976/, 812 of 899\n",
      "grabbing link /jobs/view/2638879176/, 813 of 899\n",
      "this link failed /jobs/view/2638879176/\n",
      "grabbing link /jobs/view/2650769851/, 814 of 899\n",
      "grabbing link /jobs/view/2659864947/, 815 of 899\n",
      "grabbing link /jobs/view/2636832939/, 816 of 899\n",
      "grabbing link /jobs/view/2635934604/, 817 of 899\n",
      "grabbing link /jobs/view/2658889751/, 818 of 899\n",
      "grabbing link /jobs/view/2661219770/, 819 of 899\n",
      "this link failed /jobs/view/2661219770/\n",
      "grabbing link /jobs/view/2631342998/, 820 of 899\n",
      "this link failed /jobs/view/2631342998/\n",
      "grabbing link /jobs/view/2667359847/, 821 of 899\n",
      "grabbing link /jobs/view/2660761855/, 822 of 899\n",
      "grabbing link /jobs/view/2643981353/, 823 of 899\n",
      "grabbing link /jobs/view/2654739750/, 824 of 899\n",
      "this link failed /jobs/view/2654739750/\n",
      "grabbing link /jobs/view/2659938965/, 825 of 899\n",
      "grabbing link /jobs/view/2655865562/, 826 of 899\n",
      "this link failed /jobs/view/2655865562/\n",
      "grabbing link /jobs/view/2660763534/, 827 of 899\n",
      "grabbing link /jobs/view/2659812879/, 828 of 899\n",
      "grabbing link /jobs/view/2424249013/, 829 of 899\n",
      "grabbing link /jobs/view/2649012680/, 830 of 899\n",
      "grabbing link /jobs/view/2643981357/, 831 of 899\n",
      "grabbing link /jobs/view/2424242876/, 832 of 899\n",
      "this link failed /jobs/view/2424242876/\n",
      "grabbing link /jobs/view/2654407263/, 833 of 899\n",
      "grabbing link /jobs/view/2663433661/, 834 of 899\n",
      "this link failed /jobs/view/2663433661/\n",
      "grabbing link /jobs/view/2660823895/, 835 of 899\n",
      "this link failed /jobs/view/2660823895/\n",
      "grabbing link /jobs/view/2619836967/, 836 of 899\n",
      "grabbing link /jobs/view/2394772634/, 837 of 899\n",
      "this link failed /jobs/view/2394772634/\n",
      "grabbing link /jobs/view/2512373251/, 838 of 899\n",
      "this link failed /jobs/view/2512373251/\n",
      "grabbing link /jobs/view/2639852752/, 839 of 899\n",
      "grabbing link /jobs/view/2641602661/, 840 of 899\n",
      "grabbing link /jobs/view/2632922648/, 841 of 899\n",
      "grabbing link /jobs/view/2644467922/, 842 of 899\n",
      "this link failed /jobs/view/2644467922/\n",
      "grabbing link /jobs/view/2658235406/, 843 of 899\n",
      "grabbing link /jobs/view/2656343211/, 844 of 899\n",
      "grabbing link /jobs/view/2646199420/, 845 of 899\n",
      "grabbing link /jobs/view/2657973884/, 846 of 899\n",
      "this link failed /jobs/view/2657973884/\n",
      "grabbing link /jobs/view/2652069106/, 847 of 899\n",
      "this link failed /jobs/view/2652069106/\n",
      "grabbing link /jobs/view/2656393734/, 848 of 899\n",
      "grabbing link /jobs/view/2543472849/, 849 of 899\n",
      "this link failed /jobs/view/2543472849/\n",
      "grabbing link /jobs/view/2642111475/, 850 of 899\n",
      "grabbing link /jobs/view/1448885620/, 851 of 899\n",
      "this link failed /jobs/view/1448885620/\n",
      "grabbing link /jobs/view/2665657835/, 852 of 899\n",
      "this link failed /jobs/view/2665657835/\n",
      "grabbing link /jobs/view/2625867157/, 853 of 899\n",
      "grabbing link /jobs/view/2665229171/, 854 of 899\n",
      "grabbing link /jobs/view/2642095907/, 855 of 899\n",
      "this link failed /jobs/view/2642095907/\n",
      "grabbing link /jobs/view/2619858164/, 856 of 899\n",
      "grabbing link /jobs/view/2667347339/, 857 of 899\n",
      "this link failed /jobs/view/2667347339/\n",
      "grabbing link /jobs/view/2660735127/, 858 of 899\n",
      "grabbing link /jobs/view/2641607021/, 859 of 899\n",
      "this link failed /jobs/view/2641607021/\n",
      "grabbing link /jobs/view/2654617182/, 860 of 899\n",
      "this link failed /jobs/view/2654617182/\n",
      "grabbing link /jobs/view/2477296766/, 861 of 899\n",
      "grabbing link /jobs/view/2619858262/, 862 of 899\n",
      "this link failed /jobs/view/2619858262/\n",
      "grabbing link /jobs/view/2654475085/, 863 of 899\n",
      "grabbing link /jobs/view/2511261264/, 864 of 899\n",
      "grabbing link /jobs/view/2654612941/, 865 of 899\n",
      "grabbing link /jobs/view/2639121890/, 866 of 899\n",
      "this link failed /jobs/view/2639121890/\n",
      "grabbing link /jobs/view/2642083359/, 867 of 899\n",
      "grabbing link /jobs/view/2588423755/, 868 of 899\n",
      "grabbing link /jobs/view/2659084299/, 869 of 899\n",
      "grabbing link /jobs/view/2659088142/, 870 of 899\n",
      "grabbing link /jobs/view/2588177238/, 871 of 899\n",
      "grabbing link /jobs/view/2643313906/, 872 of 899\n",
      "this link failed /jobs/view/2643313906/\n",
      "grabbing link /jobs/view/2659225232/, 873 of 899\n",
      "this link failed /jobs/view/2659225232/\n",
      "grabbing link /jobs/view/2650782889/, 874 of 899\n",
      "grabbing link /jobs/view/2618756017/, 875 of 899\n",
      "this link failed /jobs/view/2618756017/\n",
      "grabbing link /jobs/view/2666496914/, 876 of 899\n",
      "this link failed /jobs/view/2666496914/\n",
      "grabbing link /jobs/view/2486220651/, 877 of 899\n",
      "grabbing link /jobs/view/2643682389/, 878 of 899\n",
      "this link failed /jobs/view/2643682389/\n",
      "grabbing link /jobs/view/2658214426/, 879 of 899\n",
      "grabbing link /jobs/view/2582613069/, 880 of 899\n",
      "grabbing link /jobs/view/2650785207/, 881 of 899\n",
      "grabbing link /jobs/view/2657392025/, 882 of 899\n",
      "grabbing link /jobs/view/2654398035/, 883 of 899\n",
      "grabbing link /jobs/view/2625606920/, 884 of 899\n",
      "grabbing link /jobs/view/2660719650/, 885 of 899\n",
      "grabbing link /jobs/view/2656183338/, 886 of 899\n",
      "grabbing link /jobs/view/2636815319/, 887 of 899\n",
      "this link failed /jobs/view/2636815319/\n",
      "grabbing link /jobs/view/2639380260/, 888 of 899\n",
      "grabbing link /jobs/view/2598024329/, 889 of 899\n",
      "this link failed /jobs/view/2598024329/\n",
      "grabbing link /jobs/view/2656850001/, 890 of 899\n",
      "grabbing link /jobs/view/2658961458/, 891 of 899\n",
      "grabbing link /jobs/view/2656848269/, 892 of 899\n",
      "grabbing link /jobs/view/2646777680/, 893 of 899\n",
      "grabbing link /jobs/view/2658227025/, 894 of 899\n",
      "this link failed /jobs/view/2658227025/\n",
      "grabbing link /jobs/view/2654576391/, 895 of 899\n",
      "grabbing link /jobs/view/2552490103/, 896 of 899\n",
      "this link failed /jobs/view/2552490103/\n",
      "grabbing link /jobs/view/2659511685/, 897 of 899\n",
      "grabbing link /jobs/view/2638506391/, 898 of 899\n",
      "grabbing link /jobs/view/2657052027/, 899 of 899\n",
      "this link failed /jobs/view/2657052027/\n",
      "CPU times: user 1min 14s, sys: 2.78 s, total: 1min 17s\n",
      "Wall time: 1h 3min 18s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "      <th>Job URL</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Seniority Level</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Job Function</th>\n",
       "      <th>Industries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Associate Solution Engineer - Sydney</td>\n",
       "      <td>Appian Corporation</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2622582377/</td>\n",
       "      <td>At Appian, we’re seeking to grow our culture ...</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Presales solution Engineer</td>\n",
       "      <td>Powell Software</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2649187443/</td>\n",
       "      <td></td>\n",
       "      <td>Associate</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Sales and Information Technology</td>\n",
       "      <td>Information Technology and Services</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Service Cloud Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2652778606/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Associate Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2661212338/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2661268390/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>RAMS Engineer</td>\n",
       "      <td>Jobs on jobactive</td>\n",
       "      <td>Docklands, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2659088142/</td>\n",
       "      <td>We currently have a career defining opportuni...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Restaurants, Food &amp; Beverages, and Design</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>Field Services Engineer - Mechanical (Turbomac...</td>\n",
       "      <td>Baker Hughes</td>\n",
       "      <td>Moomba, South Australia, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2588177238/</td>\n",
       "      <td>This is not an active position, this posting ...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Information Technology and Services, Computer ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>Mechanical Engineer</td>\n",
       "      <td>Wave International Pty Ltd</td>\n",
       "      <td>Brisbane, Queensland, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2650782889/</td>\n",
       "      <td>Leading Engineering Consultancy working from ...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Staffing and Recruiting, Oil &amp; Energy, and Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>Dynamics 365 F&amp;O Solution Architect</td>\n",
       "      <td>Microsoft</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2486220651/</td>\n",
       "      <td>The Microsoft Asia Business Application Domai...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Hardware, Computer Software, and Info...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>Technical Support Engineer</td>\n",
       "      <td>Agilent Technologies</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2658214426/</td>\n",
       "      <td>Agilent inspires and supports discoveries tha...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Research and Biotechnology</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>566 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Job Title  \\\n",
       "0                 Associate Solution Engineer - Sydney   \n",
       "1                           Presales solution Engineer   \n",
       "2                      Service Cloud Solution Engineer   \n",
       "3                          Associate Solution Engineer   \n",
       "4                                    Solution Engineer   \n",
       "..                                                 ...   \n",
       "561                                      RAMS Engineer   \n",
       "562  Field Services Engineer - Mechanical (Turbomac...   \n",
       "563                                Mechanical Engineer   \n",
       "564                Dynamics 365 F&O Solution Architect   \n",
       "565                         Technical Support Engineer   \n",
       "\n",
       "                        Company                        Job Location  \\\n",
       "0            Appian Corporation  Sydney, New South Wales, Australia   \n",
       "1               Powell Software  Sydney, New South Wales, Australia   \n",
       "2                    Salesforce      Melbourne, Victoria, Australia   \n",
       "3                    Salesforce  Sydney, New South Wales, Australia   \n",
       "4                    Salesforce  Sydney, New South Wales, Australia   \n",
       "..                          ...                                 ...   \n",
       "561           Jobs on jobactive      Docklands, Victoria, Australia   \n",
       "562                Baker Hughes  Moomba, South Australia, Australia   \n",
       "563  Wave International Pty Ltd     Brisbane, Queensland, Australia   \n",
       "564                   Microsoft      Melbourne, Victoria, Australia   \n",
       "565        Agilent Technologies      Melbourne, Victoria, Australia   \n",
       "\n",
       "                                            Job URL  \\\n",
       "0    https://www.linkedin.com/jobs/view/2622582377/   \n",
       "1    https://www.linkedin.com/jobs/view/2649187443/   \n",
       "2    https://www.linkedin.com/jobs/view/2652778606/   \n",
       "3    https://www.linkedin.com/jobs/view/2661212338/   \n",
       "4    https://www.linkedin.com/jobs/view/2661268390/   \n",
       "..                                              ...   \n",
       "561  https://www.linkedin.com/jobs/view/2659088142/   \n",
       "562  https://www.linkedin.com/jobs/view/2588177238/   \n",
       "563  https://www.linkedin.com/jobs/view/2650782889/   \n",
       "564  https://www.linkedin.com/jobs/view/2486220651/   \n",
       "565  https://www.linkedin.com/jobs/view/2658214426/   \n",
       "\n",
       "                                       Job Description Seniority Level  \\\n",
       "0     At Appian, we’re seeking to grow our culture ...       Associate   \n",
       "1                                                            Associate   \n",
       "2     To get the best candidate experience, please ...  Not Applicable   \n",
       "3     To get the best candidate experience, please ...  Not Applicable   \n",
       "4     To get the best candidate experience, please ...  Not Applicable   \n",
       "..                                                 ...             ...   \n",
       "561   We currently have a career defining opportuni...     Entry level   \n",
       "562   This is not an active position, this posting ...     Entry level   \n",
       "563   Leading Engineering Consultancy working from ...     Entry level   \n",
       "564   The Microsoft Asia Business Application Domai...  Not Applicable   \n",
       "565   Agilent inspires and supports discoveries tha...     Entry level   \n",
       "\n",
       "    Employment Type                            Job Function  \\\n",
       "0         Full-time  Engineering and Information Technology   \n",
       "1         Full-time        Sales and Information Technology   \n",
       "2         Full-time  Engineering and Information Technology   \n",
       "3         Full-time  Engineering and Information Technology   \n",
       "4         Full-time  Engineering and Information Technology   \n",
       "..              ...                                     ...   \n",
       "561       Full-time  Engineering and Information Technology   \n",
       "562       Full-time                  Information Technology   \n",
       "563       Full-time  Engineering and Information Technology   \n",
       "564       Full-time  Engineering and Information Technology   \n",
       "565       Full-time                  Information Technology   \n",
       "\n",
       "                                            Industries  \n",
       "0                                    Computer Software  \n",
       "1                  Information Technology and Services  \n",
       "2    Computer Software, Information Technology and ...  \n",
       "3    Computer Software, Information Technology and ...  \n",
       "4    Computer Software, Information Technology and ...  \n",
       "..                                                 ...  \n",
       "561          Restaurants, Food & Beverages, and Design  \n",
       "562  Information Technology and Services, Computer ...  \n",
       "563  Staffing and Recruiting, Oil & Energy, and Man...  \n",
       "564  Computer Hardware, Computer Software, and Info...  \n",
       "565                         Research and Biotechnology  \n",
       "\n",
       "[566 rows x 9 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# making the first good data frame\n",
    "# login to linkedin with cookie\n",
    "driver = login_with_cookie(cookie_path='linkedin_cookies_2021-02-14.pkl')\n",
    "# use selenium to copy all job links\n",
    "# https://www.linkedin.com/jobs/search/?f_PP=104769905%2C100992797&geoId=101452733&keywords=machine%20learning%20engineer&location=Australia&sortBy=R\n",
    "keywords = 'solution engineer'\n",
    "location = 'Australia'\n",
    "all_job_links = search_grab_links_v3(driver=driver,kws=keywords,loc=location,top_pages='all')\n",
    "# use requests and gazpacho to get all the details of a job\n",
    "jobs = grab_job_descs_v2(all_job_links)\n",
    "# make the list into a dataframe\n",
    "jobs_df = joblist_to_df(jobs)\n",
    "# drop the duplicates -- linkedin displays the same job from different recruiters sometimes\n",
    "jobs_df = jobs_df.drop_duplicates(subset=['Job Title', 'Company', 'Job Location', 'Job Description',\n",
    "       'Seniority Level', 'Employment Type', 'Job Function', 'Industries'])\n",
    "jobs_df = jobs_df.reset_index(drop=True)\n",
    "jobs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the li[number] will do for now for a quick fix\n",
    "# driver.find_element_by_xpath('/html/body/div[5]/div[3]/div[3]/div/div/section[1]/div/div/section/div/ul/li[9]/button').click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'all_job_links' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'all_job_links' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "jobs = grab_job_descs_v2(all_job_links)\n",
    "# make the list into a dataframe\n",
    "jobs_df = joblist_to_df(jobs)\n",
    "# drop the duplicates -- linkedin displays the same job from different recruiters sometimes\n",
    "jobs_df = jobs_df.drop_duplicates(subset=['Job Title', 'Company', 'Job Location', 'Job Description',\n",
    "       'Seniority Level', 'Employment Type', 'Job Function', 'Industries'])\n",
    "jobs_df = jobs_df.reset_index(drop=True)\n",
    "jobs_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now we have our dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "      <th>Job URL</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Seniority Level</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Job Function</th>\n",
       "      <th>Industries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data &amp; Quality Manager</td>\n",
       "      <td>Metcash</td>\n",
       "      <td>Macquarie Park, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2658781767/</td>\n",
       "      <td>Join an award winning team who have created o...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Food &amp; Beverages, Consumer Goods, and Retail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Senior Manager Digital Insights Analytics</td>\n",
       "      <td>Commonwealth Bank</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2651938665/</td>\n",
       "      <td>Senior ManagerData and Decision Science(12 mo...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Research, Analyst, and Information Technology</td>\n",
       "      <td>Information Technology and Services, Banking, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Analytics and Insights Manager</td>\n",
       "      <td>Domain</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2615561143/</td>\n",
       "      <td>Analytics and Insights Manager - Sydney - Per...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Analyst, Finance, and Research</td>\n",
       "      <td>Information Technology and Services, Online Me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data and Analytics Manager</td>\n",
       "      <td>Macquarie Group</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2645644230/</td>\n",
       "      <td>Data and Analytics ManagerSydney Are you a Da...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Banking, Financial Services, and Investment Ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Analytics &amp; Insights Manager</td>\n",
       "      <td>Nine</td>\n",
       "      <td>North Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2650968303/</td>\n",
       "      <td>Department/Team OverviewThe Product team is r...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Online Media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>Marketing Analytics Manager</td>\n",
       "      <td>Sbs</td>\n",
       "      <td>New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2644918260/</td>\n",
       "      <td>There’s a reason people work at SBS. SBS is o...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Research, Analyst, and Information Technology</td>\n",
       "      <td>Marketing and Advertising, Online Media, and B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>EPMO Portfolio Financial &amp; Reporting Manager</td>\n",
       "      <td>Optus</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2608215470/</td>\n",
       "      <td>Optus’ vision is to be Australia’s most loved...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Accounting/Auditing and Finance</td>\n",
       "      <td>Telecommunications, Internet, and Information ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>Analyst - Quantitative Modelling</td>\n",
       "      <td>Bank of Queensland</td>\n",
       "      <td>Newstead, Queensland, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2520437408/</td>\n",
       "      <td>Join an ASX 100 finance leader! Exciting grow...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Business Development and Sales</td>\n",
       "      <td>Banking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>ANZ</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2645676830/</td>\n",
       "      <td>Data Scientist - Customer Resolutions Locatio...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Accounting, Banking, and Financial Services</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>Credit Cards Product Manager, Citi Consumer Bank</td>\n",
       "      <td>Citi</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2612612684/</td>\n",
       "      <td>Job Id: 21296654 Product Manager – Coles Cred...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Product Management and Marketing</td>\n",
       "      <td>Banking, Financial Services, and Investment Ba...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>484 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Job Title             Company  \\\n",
       "0                              Data & Quality Manager             Metcash   \n",
       "1           Senior Manager Digital Insights Analytics   Commonwealth Bank   \n",
       "2                      Analytics and Insights Manager              Domain   \n",
       "3                          Data and Analytics Manager     Macquarie Group   \n",
       "4                        Analytics & Insights Manager                Nine   \n",
       "..                                                ...                 ...   \n",
       "479                       Marketing Analytics Manager                 Sbs   \n",
       "480      EPMO Portfolio Financial & Reporting Manager               Optus   \n",
       "481                  Analyst - Quantitative Modelling  Bank of Queensland   \n",
       "482                                    Data Scientist                 ANZ   \n",
       "483  Credit Cards Product Manager, Citi Consumer Bank                Citi   \n",
       "\n",
       "                                   Job Location  \\\n",
       "0    Macquarie Park, New South Wales, Australia   \n",
       "1            Sydney, New South Wales, Australia   \n",
       "2            Sydney, New South Wales, Australia   \n",
       "3            Sydney, New South Wales, Australia   \n",
       "4      North Sydney, New South Wales, Australia   \n",
       "..                                          ...   \n",
       "479                  New South Wales, Australia   \n",
       "480          Sydney, New South Wales, Australia   \n",
       "481             Newstead, Queensland, Australia   \n",
       "482              Melbourne, Victoria, Australia   \n",
       "483          Sydney, New South Wales, Australia   \n",
       "\n",
       "                                            Job URL  \\\n",
       "0    https://www.linkedin.com/jobs/view/2658781767/   \n",
       "1    https://www.linkedin.com/jobs/view/2651938665/   \n",
       "2    https://www.linkedin.com/jobs/view/2615561143/   \n",
       "3    https://www.linkedin.com/jobs/view/2645644230/   \n",
       "4    https://www.linkedin.com/jobs/view/2650968303/   \n",
       "..                                              ...   \n",
       "479  https://www.linkedin.com/jobs/view/2644918260/   \n",
       "480  https://www.linkedin.com/jobs/view/2608215470/   \n",
       "481  https://www.linkedin.com/jobs/view/2520437408/   \n",
       "482  https://www.linkedin.com/jobs/view/2645676830/   \n",
       "483  https://www.linkedin.com/jobs/view/2612612684/   \n",
       "\n",
       "                                       Job Description   Seniority Level  \\\n",
       "0     Join an award winning team who have created o...  Mid-Senior level   \n",
       "1     Senior ManagerData and Decision Science(12 mo...  Mid-Senior level   \n",
       "2     Analytics and Insights Manager - Sydney - Per...  Mid-Senior level   \n",
       "3     Data and Analytics ManagerSydney Are you a Da...    Not Applicable   \n",
       "4     Department/Team OverviewThe Product team is r...  Mid-Senior level   \n",
       "..                                                 ...               ...   \n",
       "479   There’s a reason people work at SBS. SBS is o...  Mid-Senior level   \n",
       "480   Optus’ vision is to be Australia’s most loved...  Mid-Senior level   \n",
       "481   Join an ASX 100 finance leader! Exciting grow...    Not Applicable   \n",
       "482   Data Scientist - Customer Resolutions Locatio...    Not Applicable   \n",
       "483   Job Id: 21296654 Product Manager – Coles Cred...    Not Applicable   \n",
       "\n",
       "    Employment Type                                   Job Function  \\\n",
       "0         Full-time                         Information Technology   \n",
       "1         Full-time  Research, Analyst, and Information Technology   \n",
       "2         Full-time                 Analyst, Finance, and Research   \n",
       "3         Full-time                                        Finance   \n",
       "4         Full-time                         Information Technology   \n",
       "..              ...                                            ...   \n",
       "479       Full-time  Research, Analyst, and Information Technology   \n",
       "480       Full-time                Accounting/Auditing and Finance   \n",
       "481       Full-time                 Business Development and Sales   \n",
       "482       Full-time         Engineering and Information Technology   \n",
       "483       Full-time               Product Management and Marketing   \n",
       "\n",
       "                                            Industries  \n",
       "0         Food & Beverages, Consumer Goods, and Retail  \n",
       "1    Information Technology and Services, Banking, ...  \n",
       "2    Information Technology and Services, Online Me...  \n",
       "3    Banking, Financial Services, and Investment Ba...  \n",
       "4                                         Online Media  \n",
       "..                                                 ...  \n",
       "479  Marketing and Advertising, Online Media, and B...  \n",
       "480  Telecommunications, Internet, and Information ...  \n",
       "481                                            Banking  \n",
       "482        Accounting, Banking, and Financial Services  \n",
       "483  Banking, Financial Services, and Investment Ba...  \n",
       "\n",
       "[484 rows x 9 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating the final tables begins\n",
    "\n",
    "## Feature Engineering for our Raw Data - Adding Salary and fixing some locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "filter_dollars = jobs_df['Job Description'].str.contains('\\$')\n",
    "# the below styles should be tailored for your country\n",
    "# salary style1 $100,000 or $123456\n",
    "reg_salary1 = '\\$\\d{2,3},{0,1}\\d{3}'\n",
    "filter_salary1 = jobs_df['Job Description'].str.contains(reg_salary1)\n",
    "# salaray style2 = $100k or $123 k\n",
    "reg_salary2 = '\\$\\d{2,3} {0,1}k'\n",
    "filter_salary2 = jobs_df['Job Description'].str.contains(reg_salary2)\n",
    "# jobs_df[filter_dollars]['Job Description'].str.extract('(\\$\\d{2,3},{0,1}\\d{3})')\n",
    "# jobs_df['Job Description'].str.extract('(\\$\\d{2,3} {0,1}k)')\n",
    "# jobs_df[filter_dollars]['Job Description'].iloc[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# based on the patterns that salaries are posted in\n",
    "# should be tailored for your country\n",
    "# salary style1 $100,000 or $123456\n",
    "# salaray style2 = $100k or $123 k\n",
    "def grab_salary(jobs_df,salary_patterns):\n",
    "    # the broadest filter, if they have a $ sign, should change this symbol for your own country\n",
    "    # alternatively the broadest filter can be the word \"salary\" or \"wage\", etc\n",
    "    filter_dollars = jobs_df['Job Description'].str.contains('\\$')\n",
    "    temp_output = []\n",
    "    for pattern in salary_patterns:\n",
    "        # setup the filter first\n",
    "        filter_salary = jobs_df['Job Description'].str.contains(pattern)\n",
    "        # create the df and extract all from that pattern\n",
    "        temp_df = jobs_df[filter_salary]['Job Description'].str.extractall(f'({pattern})')\n",
    "        # unstack puts all results in a new col rather but is still multiindex\n",
    "        temp_df = temp_df.unstack()\n",
    "        # this makes all salaries into style1\n",
    "        temp_df = temp_df.apply(lambda x: x.str.replace('k',',000'))\n",
    "        # remove the redundant multiindex\n",
    "        temp_df.columns = temp_df.columns.droplevel()\n",
    "        # add it to our list\n",
    "        temp_output.append(temp_df)\n",
    "    \n",
    "    # return the concat of our lists\n",
    "    return pd.concat(temp_output,axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# do some cleaning on our salaries\n",
    "# mainly because companies sometimes put a single salary \n",
    "# and sometimes put a salary range\n",
    "# we want to compare average salaries between companies\n",
    "def fix_salary(salary_df):\n",
    "    # remove duplicates if a company somehow put both salary forms\n",
    "    salary_df = salary_df.reset_index().drop_duplicates(subset='index').set_index('index')\n",
    "    # remove the names of the columns and index\n",
    "    salary_df.index.name=None\n",
    "    salary_df.columns.name=None\n",
    "    # make a dataframe of numbers so we can do math on it\n",
    "    salary_df_num = salary_df.apply(lambda x: pd.to_numeric(x.str.replace('\\$|,',''),errors='ignore',downcast='integer'))\n",
    "    # min, max and avg salary in numbers\n",
    "    salary_df['Min_Salary_Num'] = salary_df_num.min(axis='columns')\n",
    "    salary_df['Max_Salary_Num'] = salary_df_num.max(axis='columns')\n",
    "    salary_df['Avg_Salary_Num'] = salary_df[['Min_Salary_Num','Max_Salary_Num']].mean(axis='columns')\n",
    "    # the 3 cols in nice number format\n",
    "    salary_df['Min_Salary'] = salary_df['Min_Salary_Num'].apply(lambda x: '${:,.0f}'.format(x))\n",
    "    salary_df['Max_Salary'] = salary_df['Max_Salary_Num'].apply(lambda x: '${:,.0f}'.format(x))\n",
    "    salary_df['Avg_Salary'] = salary_df['Avg_Salary_Num'].apply(lambda x: '${:,.0f}'.format(x))\n",
    "    # remove our intermediary columns\n",
    "    return salary_df[['Min_Salary','Max_Salary','Avg_Salary','Min_Salary_Num','Max_Salary_Num','Avg_Salary_Num']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Min_Salary</th>\n",
       "      <th>Max_Salary</th>\n",
       "      <th>Avg_Salary</th>\n",
       "      <th>Min_Salary_Num</th>\n",
       "      <th>Max_Salary_Num</th>\n",
       "      <th>Avg_Salary_Num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>$119,442</td>\n",
       "      <td>$119,442</td>\n",
       "      <td>$119,442</td>\n",
       "      <td>119442.0</td>\n",
       "      <td>119442.0</td>\n",
       "      <td>119442.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>$250,000</td>\n",
       "      <td>$250,000</td>\n",
       "      <td>$250,000</td>\n",
       "      <td>250000.0</td>\n",
       "      <td>250000.0</td>\n",
       "      <td>250000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>$100,000</td>\n",
       "      <td>$100,000</td>\n",
       "      <td>$100,000</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>100000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>$100,000</td>\n",
       "      <td>$140,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>120000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>111077.0</td>\n",
       "      <td>111077.0</td>\n",
       "      <td>111077.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>$130,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "      <td>130000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>$175,000</td>\n",
       "      <td>$175,000</td>\n",
       "      <td>$175,000</td>\n",
       "      <td>175000.0</td>\n",
       "      <td>175000.0</td>\n",
       "      <td>175000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>111077.0</td>\n",
       "      <td>111077.0</td>\n",
       "      <td>111077.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>$60,000</td>\n",
       "      <td>$80,000</td>\n",
       "      <td>$70,000</td>\n",
       "      <td>60000.0</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>70000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>$120,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>120000.0</td>\n",
       "      <td>120000.0</td>\n",
       "      <td>120000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>$120,000</td>\n",
       "      <td>$140,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>120000.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>130000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Min_Salary Max_Salary Avg_Salary  Min_Salary_Num  Max_Salary_Num  \\\n",
       "53    $119,442   $119,442   $119,442        119442.0        119442.0   \n",
       "180   $250,000   $250,000   $250,000        250000.0        250000.0   \n",
       "233   $100,000   $100,000   $100,000        100000.0        100000.0   \n",
       "242   $100,000   $140,000   $120,000        100000.0        140000.0   \n",
       "260   $111,077   $111,077   $111,077        111077.0        111077.0   \n",
       "292   $130,000   $130,000   $130,000        130000.0        130000.0   \n",
       "394   $175,000   $175,000   $175,000        175000.0        175000.0   \n",
       "456   $111,077   $111,077   $111,077        111077.0        111077.0   \n",
       "475    $60,000    $80,000    $70,000         60000.0         80000.0   \n",
       "500   $120,000   $120,000   $120,000        120000.0        120000.0   \n",
       "510   $120,000   $140,000   $130,000        120000.0        140000.0   \n",
       "\n",
       "     Avg_Salary_Num  \n",
       "53         119442.0  \n",
       "180        250000.0  \n",
       "233        100000.0  \n",
       "242        120000.0  \n",
       "260        111077.0  \n",
       "292        130000.0  \n",
       "394        175000.0  \n",
       "456        111077.0  \n",
       "475         70000.0  \n",
       "500        120000.0  \n",
       "510        130000.0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df = fix_salary(grab_salary(jobs_df,\n",
    "                                   salary_patterns= [\n",
    "                                       '\\$\\d{2,3},{0,1}\\d{3}',\n",
    "                                       '\\$\\d{2,3} {0,1}k'\n",
    "                                   ]))\n",
    "salary_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter_state = jobs_df['Job Location'].str.contains('united',case=False)\n",
    "# jobs_df['Job Location'][filter_state]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "      <th>Job URL</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Seniority Level</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Job Function</th>\n",
       "      <th>Industries</th>\n",
       "      <th>Min_Salary</th>\n",
       "      <th>Max_Salary</th>\n",
       "      <th>Avg_Salary</th>\n",
       "      <th>Min_Salary_Num</th>\n",
       "      <th>Max_Salary_Num</th>\n",
       "      <th>Avg_Salary_Num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Associate Solution Engineer - Sydney</td>\n",
       "      <td>Appian Corporation</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2622582377/</td>\n",
       "      <td>At Appian, we’re seeking to grow our culture ...</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Presales solution Engineer</td>\n",
       "      <td>Powell Software</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2649187443/</td>\n",
       "      <td></td>\n",
       "      <td>Associate</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Sales and Information Technology</td>\n",
       "      <td>Information Technology and Services</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Service Cloud Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2652778606/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Associate Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2661212338/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Solution Engineer</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2661268390/</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>RAMS Engineer</td>\n",
       "      <td>Jobs on jobactive</td>\n",
       "      <td>Docklands, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2659088142/</td>\n",
       "      <td>We currently have a career defining opportuni...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Restaurants, Food &amp; Beverages, and Design</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>Field Services Engineer - Mechanical (Turbomac...</td>\n",
       "      <td>Baker Hughes</td>\n",
       "      <td>Moomba, South Australia, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2588177238/</td>\n",
       "      <td>This is not an active position, this posting ...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Information Technology and Services, Computer ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>Mechanical Engineer</td>\n",
       "      <td>Wave International Pty Ltd</td>\n",
       "      <td>Brisbane, Queensland, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2650782889/</td>\n",
       "      <td>Leading Engineering Consultancy working from ...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Staffing and Recruiting, Oil &amp; Energy, and Man...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>Dynamics 365 F&amp;O Solution Architect</td>\n",
       "      <td>Microsoft</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2486220651/</td>\n",
       "      <td>The Microsoft Asia Business Application Domai...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Engineering and Information Technology</td>\n",
       "      <td>Computer Hardware, Computer Software, and Info...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>Technical Support Engineer</td>\n",
       "      <td>Agilent Technologies</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2658214426/</td>\n",
       "      <td>Agilent inspires and supports discoveries tha...</td>\n",
       "      <td>Entry level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Research and Biotechnology</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>566 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Job Title  \\\n",
       "0                 Associate Solution Engineer - Sydney   \n",
       "1                           Presales solution Engineer   \n",
       "2                      Service Cloud Solution Engineer   \n",
       "3                          Associate Solution Engineer   \n",
       "4                                    Solution Engineer   \n",
       "..                                                 ...   \n",
       "561                                      RAMS Engineer   \n",
       "562  Field Services Engineer - Mechanical (Turbomac...   \n",
       "563                                Mechanical Engineer   \n",
       "564                Dynamics 365 F&O Solution Architect   \n",
       "565                         Technical Support Engineer   \n",
       "\n",
       "                        Company                        Job Location  \\\n",
       "0            Appian Corporation  Sydney, New South Wales, Australia   \n",
       "1               Powell Software  Sydney, New South Wales, Australia   \n",
       "2                    Salesforce      Melbourne, Victoria, Australia   \n",
       "3                    Salesforce  Sydney, New South Wales, Australia   \n",
       "4                    Salesforce  Sydney, New South Wales, Australia   \n",
       "..                          ...                                 ...   \n",
       "561           Jobs on jobactive      Docklands, Victoria, Australia   \n",
       "562                Baker Hughes  Moomba, South Australia, Australia   \n",
       "563  Wave International Pty Ltd     Brisbane, Queensland, Australia   \n",
       "564                   Microsoft      Melbourne, Victoria, Australia   \n",
       "565        Agilent Technologies      Melbourne, Victoria, Australia   \n",
       "\n",
       "                                            Job URL  \\\n",
       "0    https://www.linkedin.com/jobs/view/2622582377/   \n",
       "1    https://www.linkedin.com/jobs/view/2649187443/   \n",
       "2    https://www.linkedin.com/jobs/view/2652778606/   \n",
       "3    https://www.linkedin.com/jobs/view/2661212338/   \n",
       "4    https://www.linkedin.com/jobs/view/2661268390/   \n",
       "..                                              ...   \n",
       "561  https://www.linkedin.com/jobs/view/2659088142/   \n",
       "562  https://www.linkedin.com/jobs/view/2588177238/   \n",
       "563  https://www.linkedin.com/jobs/view/2650782889/   \n",
       "564  https://www.linkedin.com/jobs/view/2486220651/   \n",
       "565  https://www.linkedin.com/jobs/view/2658214426/   \n",
       "\n",
       "                                       Job Description Seniority Level  \\\n",
       "0     At Appian, we’re seeking to grow our culture ...       Associate   \n",
       "1                                                            Associate   \n",
       "2     To get the best candidate experience, please ...  Not Applicable   \n",
       "3     To get the best candidate experience, please ...  Not Applicable   \n",
       "4     To get the best candidate experience, please ...  Not Applicable   \n",
       "..                                                 ...             ...   \n",
       "561   We currently have a career defining opportuni...     Entry level   \n",
       "562   This is not an active position, this posting ...     Entry level   \n",
       "563   Leading Engineering Consultancy working from ...     Entry level   \n",
       "564   The Microsoft Asia Business Application Domai...  Not Applicable   \n",
       "565   Agilent inspires and supports discoveries tha...     Entry level   \n",
       "\n",
       "    Employment Type                            Job Function  \\\n",
       "0         Full-time  Engineering and Information Technology   \n",
       "1         Full-time        Sales and Information Technology   \n",
       "2         Full-time  Engineering and Information Technology   \n",
       "3         Full-time  Engineering and Information Technology   \n",
       "4         Full-time  Engineering and Information Technology   \n",
       "..              ...                                     ...   \n",
       "561       Full-time  Engineering and Information Technology   \n",
       "562       Full-time                  Information Technology   \n",
       "563       Full-time  Engineering and Information Technology   \n",
       "564       Full-time  Engineering and Information Technology   \n",
       "565       Full-time                  Information Technology   \n",
       "\n",
       "                                            Industries Min_Salary Max_Salary  \\\n",
       "0                                    Computer Software                         \n",
       "1                  Information Technology and Services                         \n",
       "2    Computer Software, Information Technology and ...                         \n",
       "3    Computer Software, Information Technology and ...                         \n",
       "4    Computer Software, Information Technology and ...                         \n",
       "..                                                 ...        ...        ...   \n",
       "561          Restaurants, Food & Beverages, and Design                         \n",
       "562  Information Technology and Services, Computer ...                         \n",
       "563  Staffing and Recruiting, Oil & Energy, and Man...                         \n",
       "564  Computer Hardware, Computer Software, and Info...                         \n",
       "565                         Research and Biotechnology                         \n",
       "\n",
       "    Avg_Salary Min_Salary_Num Max_Salary_Num Avg_Salary_Num  \n",
       "0                                                            \n",
       "1                                                            \n",
       "2                                                            \n",
       "3                                                            \n",
       "4                                                            \n",
       "..         ...            ...            ...            ...  \n",
       "561                                                          \n",
       "562                                                          \n",
       "563                                                          \n",
       "564                                                          \n",
       "565                                                          \n",
       "\n",
       "[566 rows x 15 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_df1 = pd.concat([jobs_df,salary_df],axis='columns')\n",
    "# make a copy of this for our raw data\n",
    "jobs_df_withsal = jobs_df1.copy()\n",
    "# split the state and city for tableau\n",
    "# jobs_df_withsal['City'] = jobs_df_withsal['Job Location'].str.extract('^(.+),')\n",
    "# jobs_df_withsal['State'] = jobs_df_withsal['Job Location'].str.extract(',(.+)$')\n",
    "# remove all the NaNs\n",
    "jobs_df_withsal.fillna('',inplace=True)\n",
    "# make the state clean when the job puts the location in format State,Country\n",
    "# filter_state = jobs_df_withsal['State'].str.contains('united',case=False)\n",
    "# jobs_df_withsal['State'].loc[filter_state] = jobs_df_withsal['City'].loc[filter_state].apply(lambda x: us.states.lookup(x).abbr)\n",
    "today = datetime.today().strftime('%Y-%M-%d')\n",
    "jobs_df_withsal.to_csv(f'{keywords}_{location}_{today}.csv')\n",
    "jobs_df_withsal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# for the reddit post\n",
    "# use this for markdown tables https://www.tablesgenerator.com/markdown_tables\n",
    "jobs_df1['Salary_num'] = pd.to_numeric(jobs_df1['Salary'].str.replace('\\$|,|k','')) # make it a number for sorting\n",
    "jobs_df1 = jobs_df1.sort_values(by='Salary_num',ascending=False)\n",
    "filter_sal = jobs_df1['Salary'].notnull()\n",
    "filter_analyst = jobs_df1['Job Title'].str.contains('analyst',case=False)\n",
    "# make the column order a bit nicer\n",
    "col_order = ['Salary', 'Job Title',\n",
    " 'Company',\n",
    " 'Job Location',\n",
    " 'Job URL'\n",
    " ]\n",
    "jobs_sal_df = jobs_df1[filter_sal & filter_analyst][col_order]\n",
    "jobs_sal_df.to_csv('test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Min_Salary</th>\n",
       "      <th>Max_Salary</th>\n",
       "      <th>Avg_Salary</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>$250,000</td>\n",
       "      <td>$250,000</td>\n",
       "      <td>$250,000</td>\n",
       "      <td>Development Manager/Solutions Architect</td>\n",
       "      <td>FIRESOFT People</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>394</th>\n",
       "      <td>$175,000</td>\n",
       "      <td>$175,000</td>\n",
       "      <td>$175,000</td>\n",
       "      <td>Solution Architect - Digital &amp; Mobile App</td>\n",
       "      <td>Jobx Pty Ltd</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>$130,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>Solution Architect - Data &amp; Cloud Delivery</td>\n",
       "      <td>Command Recruitment Group</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>$120,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>Lead Platform Engineer - Enterprise Hcm | Peop...</td>\n",
       "      <td>Randstad Education Australia</td>\n",
       "      <td>Brisbane, Queensland, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>$120,000</td>\n",
       "      <td>$140,000</td>\n",
       "      <td>$130,000</td>\n",
       "      <td>Lead Platform Engineer - Enterprise Hcm | Peop...</td>\n",
       "      <td>Randstad Education Australia</td>\n",
       "      <td>Brisbane, Queensland, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>$119,442</td>\n",
       "      <td>$119,442</td>\n",
       "      <td>$119,442</td>\n",
       "      <td>Solution architect</td>\n",
       "      <td>City of Boroondara</td>\n",
       "      <td>Camberwell, Victoria, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>Senior Software Engineer - Multiple Roles</td>\n",
       "      <td>NSW Department of Customer Service</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>$111,077</td>\n",
       "      <td>Senior Software Engineer - Multiple Roles</td>\n",
       "      <td>I work for NSW</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>$100,000</td>\n",
       "      <td>$100,000</td>\n",
       "      <td>$100,000</td>\n",
       "      <td>Senior Solution Consultant - Hybrid Presales a...</td>\n",
       "      <td>DXC Technology</td>\n",
       "      <td>Macquarie Park, New South Wales, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>$100,000</td>\n",
       "      <td>$140,000</td>\n",
       "      <td>$120,000</td>\n",
       "      <td>Data Engineer / Sr. Data Analyst</td>\n",
       "      <td>Troocoo</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>$60,000</td>\n",
       "      <td>$80,000</td>\n",
       "      <td>$70,000</td>\n",
       "      <td>Software Engineer - Digital Solutions</td>\n",
       "      <td>Jobs on jobactive</td>\n",
       "      <td>Melbourne, Victoria, Australia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Min_Salary Max_Salary Avg_Salary  \\\n",
       "180   $250,000   $250,000   $250,000   \n",
       "394   $175,000   $175,000   $175,000   \n",
       "292   $130,000   $130,000   $130,000   \n",
       "500   $120,000   $120,000   $120,000   \n",
       "510   $120,000   $140,000   $130,000   \n",
       "53    $119,442   $119,442   $119,442   \n",
       "260   $111,077   $111,077   $111,077   \n",
       "456   $111,077   $111,077   $111,077   \n",
       "233   $100,000   $100,000   $100,000   \n",
       "242   $100,000   $140,000   $120,000   \n",
       "475    $60,000    $80,000    $70,000   \n",
       "\n",
       "                                             Job Title  \\\n",
       "180            Development Manager/Solutions Architect   \n",
       "394          Solution Architect - Digital & Mobile App   \n",
       "292         Solution Architect - Data & Cloud Delivery   \n",
       "500  Lead Platform Engineer - Enterprise Hcm | Peop...   \n",
       "510  Lead Platform Engineer - Enterprise Hcm | Peop...   \n",
       "53                                  Solution architect   \n",
       "260          Senior Software Engineer - Multiple Roles   \n",
       "456          Senior Software Engineer - Multiple Roles   \n",
       "233  Senior Solution Consultant - Hybrid Presales a...   \n",
       "242                   Data Engineer / Sr. Data Analyst   \n",
       "475              Software Engineer - Digital Solutions   \n",
       "\n",
       "                                Company  \\\n",
       "180                     FIRESOFT People   \n",
       "394                        Jobx Pty Ltd   \n",
       "292           Command Recruitment Group   \n",
       "500        Randstad Education Australia   \n",
       "510        Randstad Education Australia   \n",
       "53                   City of Boroondara   \n",
       "260  NSW Department of Customer Service   \n",
       "456                      I work for NSW   \n",
       "233                      DXC Technology   \n",
       "242                             Troocoo   \n",
       "475                   Jobs on jobactive   \n",
       "\n",
       "                                   Job Location  \n",
       "180          Sydney, New South Wales, Australia  \n",
       "394          Sydney, New South Wales, Australia  \n",
       "292          Sydney, New South Wales, Australia  \n",
       "500             Brisbane, Queensland, Australia  \n",
       "510             Brisbane, Queensland, Australia  \n",
       "53              Camberwell, Victoria, Australia  \n",
       "260          Sydney, New South Wales, Australia  \n",
       "456          Sydney, New South Wales, Australia  \n",
       "233  Macquarie Park, New South Wales, Australia  \n",
       "242              Melbourne, Victoria, Australia  \n",
       "475              Melbourne, Victoria, Australia  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_df1 = jobs_df1.sort_values(by='Min_Salary_Num',ascending=False)\n",
    "filter_sal = jobs_df1['Min_Salary_Num'].notnull()\n",
    "# make the column order a bit nicer\n",
    "col_order = ['Min_Salary','Max_Salary','Avg_Salary','Job Title','Company','Job Location',]\n",
    "jobs_sal_df = jobs_df1[filter_sal][col_order]\n",
    "jobs_sal_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding which tech stack is most in demand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import stopwords with nltk.\n",
    "from nltk.corpus import stopwords\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/jason.chiu/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# nltk doesn't come with everything\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stop is a list of stopwords\n",
    "stop = stopwords.words('english')\n",
    "newStopWords = []\n",
    "stop.extend(newStopWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# multi word tech stack\n",
    "multi_tech = {\n",
    "    'power bi':'power_bi',\n",
    "    'powerbi':'power_bi',\n",
    "    'nosql':'no_sql',\n",
    "    'no sql':'no_sql',\n",
    "    'data studio':'data_studio',\n",
    "    'datastudio':'data_studio',\n",
    "    ' r ':' _r_ ',\n",
    "    'datarobot':'data_robot',\n",
    "    'data robot':'data_robot',\n",
    "    'google cloud platform':'gcp'\n",
    "}\n",
    "darlo_multi ={\n",
    "    'adobe xd':'adobe_xd',\n",
    "    'web design':'web_design',\n",
    "    'webdesign':'web_design',\n",
    "    'mobile app':'mobile_app',\n",
    "    'mobile application':'mobile_app',\n",
    "    'product design':'product_design',\n",
    "    'user interface':'user_interface',\n",
    "    'adobe creative cloud':'adobe_creative_cloud',\n",
    "    'adobe suite':'adobe_creative_cloud',\n",
    "    'optimal sort':'optimal_sort',\n",
    "    'photoshop':'adobe_photoshop',\n",
    "    'illustrator':'adobe_illustrator',\n",
    "    'after effect':'adobe_after_effects'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "      <th>Job URL</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Seniority Level</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Job Function</th>\n",
       "      <th>Industries</th>\n",
       "      <th>Min_Salary</th>\n",
       "      <th>Max_Salary</th>\n",
       "      <th>Avg_Salary</th>\n",
       "      <th>Min_Salary_Num</th>\n",
       "      <th>Max_Salary_Num</th>\n",
       "      <th>Avg_Salary_Num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Lead UX/UI Designer</td>\n",
       "      <td>Robert Walters</td>\n",
       "      <td>Perth, Western Australia, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2645697068/</td>\n",
       "      <td>This opportunity is within a leading geoscien...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Mining &amp; Metals</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>UX Designer / Researcher</td>\n",
       "      <td>Endeavour Group</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2496656500/</td>\n",
       "      <td>About UsEndeavour Group was created in 2020 t...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Design, Art/Creative, and Information Technology</td>\n",
       "      <td>Computer Software, Retail, and Financial Services</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>UX Designer</td>\n",
       "      <td>Endeavour Group</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2642705335/</td>\n",
       "      <td>About UsEndeavour Group was created in 2020 t...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Design, Art/Creative, and Information Technology</td>\n",
       "      <td>Computer Software, Retail, and Financial Services</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Senior UX Designer</td>\n",
       "      <td>Peoplebank</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2619882773/</td>\n",
       "      <td>Location: Sydney, New South Wales Job Type: P...</td>\n",
       "      <td>Associate</td>\n",
       "      <td>Contract</td>\n",
       "      <td>Design, Art/Creative, and Information Technology</td>\n",
       "      <td>Information Technology and Services, Computer ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Senior UX Researcher / Service Designer</td>\n",
       "      <td>Objective Experience</td>\n",
       "      <td>Sydney, New South Wales, Australia</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2661207413/</td>\n",
       "      <td>We are looking for a brilliant researcher / s...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                Job Title               Company  \\\n",
       "0           0                      Lead UX/UI Designer        Robert Walters   \n",
       "1           1                 UX Designer / Researcher       Endeavour Group   \n",
       "2           2                              UX Designer       Endeavour Group   \n",
       "3           3                       Senior UX Designer            Peoplebank   \n",
       "4           4  Senior UX Researcher / Service Designer  Objective Experience   \n",
       "\n",
       "                          Job Location  \\\n",
       "0  Perth, Western Australia, Australia   \n",
       "1   Sydney, New South Wales, Australia   \n",
       "2   Sydney, New South Wales, Australia   \n",
       "3   Sydney, New South Wales, Australia   \n",
       "4   Sydney, New South Wales, Australia   \n",
       "\n",
       "                                          Job URL  \\\n",
       "0  https://www.linkedin.com/jobs/view/2645697068/   \n",
       "1  https://www.linkedin.com/jobs/view/2496656500/   \n",
       "2  https://www.linkedin.com/jobs/view/2642705335/   \n",
       "3  https://www.linkedin.com/jobs/view/2619882773/   \n",
       "4  https://www.linkedin.com/jobs/view/2661207413/   \n",
       "\n",
       "                                     Job Description   Seniority Level  \\\n",
       "0   This opportunity is within a leading geoscien...  Mid-Senior level   \n",
       "1   About UsEndeavour Group was created in 2020 t...  Mid-Senior level   \n",
       "2   About UsEndeavour Group was created in 2020 t...  Mid-Senior level   \n",
       "3   Location: Sydney, New South Wales Job Type: P...         Associate   \n",
       "4   We are looking for a brilliant researcher / s...               NaN   \n",
       "\n",
       "  Employment Type                                      Job Function  \\\n",
       "0       Full-time                            Information Technology   \n",
       "1       Full-time  Design, Art/Creative, and Information Technology   \n",
       "2       Full-time  Design, Art/Creative, and Information Technology   \n",
       "3        Contract  Design, Art/Creative, and Information Technology   \n",
       "4       Full-time                                               NaN   \n",
       "\n",
       "                                          Industries Min_Salary Max_Salary  \\\n",
       "0                                    Mining & Metals        NaN        NaN   \n",
       "1  Computer Software, Retail, and Financial Services        NaN        NaN   \n",
       "2  Computer Software, Retail, and Financial Services        NaN        NaN   \n",
       "3  Information Technology and Services, Computer ...        NaN        NaN   \n",
       "4                                                NaN        NaN        NaN   \n",
       "\n",
       "  Avg_Salary  Min_Salary_Num  Max_Salary_Num  Avg_Salary_Num  \n",
       "0        NaN             NaN             NaN             NaN  \n",
       "1        NaN             NaN             NaN             NaN  \n",
       "2        NaN             NaN             NaN             NaN  \n",
       "3        NaN             NaN             NaN             NaN  \n",
       "4        NaN             NaN             NaN             NaN  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_df = pd.read_csv('ux designer_Australia_2021-59-31.csv')\n",
    "jobs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>experience</td>\n",
       "      <td>2888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>technical</td>\n",
       "      <td>2273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>solutions</td>\n",
       "      <td>2068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>business</td>\n",
       "      <td>1883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>work</td>\n",
       "      <td>1778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>team</td>\n",
       "      <td>1593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>solution</td>\n",
       "      <td>1495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>customer</td>\n",
       "      <td>1477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>customers</td>\n",
       "      <td>1459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>skills</td>\n",
       "      <td>1259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cloud</td>\n",
       "      <td>1134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>technology</td>\n",
       "      <td>1125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>role</td>\n",
       "      <td>1022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>sales</td>\n",
       "      <td>1016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>development</td>\n",
       "      <td>1001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>data</td>\n",
       "      <td>992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>services</td>\n",
       "      <td>983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>design</td>\n",
       "      <td>965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>working</td>\n",
       "      <td>948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>support</td>\n",
       "      <td>948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>security</td>\n",
       "      <td>912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>knowledge</td>\n",
       "      <td>906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ability</td>\n",
       "      <td>863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>management</td>\n",
       "      <td>802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>people</td>\n",
       "      <td>800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>architecture</td>\n",
       "      <td>757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>including</td>\n",
       "      <td>734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>microsoft</td>\n",
       "      <td>729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>product</td>\n",
       "      <td>683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>requirements</td>\n",
       "      <td>671</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           words  count\n",
       "0     experience   2888\n",
       "1      technical   2273\n",
       "2      solutions   2068\n",
       "3       business   1883\n",
       "4           work   1778\n",
       "5           team   1593\n",
       "6       solution   1495\n",
       "7       customer   1477\n",
       "8      customers   1459\n",
       "9         skills   1259\n",
       "10         cloud   1134\n",
       "11    technology   1125\n",
       "12          role   1022\n",
       "13         sales   1016\n",
       "14   development   1001\n",
       "15          data    992\n",
       "16      services    983\n",
       "17        design    965\n",
       "18       working    948\n",
       "19       support    948\n",
       "20      security    912\n",
       "21     knowledge    906\n",
       "22       ability    863\n",
       "23    management    802\n",
       "24        people    800\n",
       "25  architecture    757\n",
       "26     including    734\n",
       "27     microsoft    729\n",
       "28       product    683\n",
       "29  requirements    671"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = jobs_df['Job Description'].to_frame()\n",
    "words.columns = ['words']\n",
    "# below regex removes any non-word, non-space characters. This will mess up some words, but we're only interested in tech we know about, so it doesn't matter much\n",
    "words['words'] = words['words'].str.replace(\"[^\\w\\s]\", \"\").str.lower()\n",
    "words['words'] = words['words'].apply(lambda x: ' '.join([item for item in x.split() if item not in stop]))\n",
    "for key in multi_tech:\n",
    "    words['words'] = words['words'].str.replace(key,multi_tech[key])\n",
    "# for key in darlo_multi:\n",
    "#     words['words'] = words['words'].str.replace(key,darlo_multi[key])\n",
    "\n",
    "freq = pd.DataFrame(words['words'].str.split(expand=True).stack().value_counts()).reset_index()\n",
    "freq.columns = ['words','count']\n",
    "freq.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>PRODUCT DESIGN</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>PRODUCT DESIGNER</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>FIGMA</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>SKETCH</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>INVISION</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>PRODUCT DESIGNERS</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>USER INTERFACE</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>CSS</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>730</th>\n",
       "      <td>JIRA</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>HTML</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 words  count\n",
       "135     PRODUCT DESIGN    104\n",
       "143   PRODUCT DESIGNER    100\n",
       "146              FIGMA     97\n",
       "186             SKETCH     81\n",
       "317           INVISION     51\n",
       "412  PRODUCT DESIGNERS     41\n",
       "418     USER INTERFACE     40\n",
       "714                CSS     23\n",
       "730               JIRA     23\n",
       "773               HTML     22"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# for love Adobe suite, Lookback, Axure, Sketch, InVision, Omnigraffle, Userzoom, Optimal sort, Figma\n",
    "# to get phrases, replace powerbi and power bi with power_bi\n",
    "tech1 = ['figma','sketch','invision','lookback',\n",
    "        'askable','miro','confluence','jira','axure',\n",
    "        'omnigraffle','userzoom','html','css','javascript']\n",
    "tech2 = tech1 + (list(set(darlo_multi.values())))\n",
    "tech = '|'.join(tech2)\n",
    "# these words contain things like aws and excel\n",
    "exclude1 = ['capital','laws','excellen']\n",
    "exclude = '|'.join(exclude1)\n",
    "\n",
    "filter3 = freq['words'].str.contains(tech)\n",
    "filter4 = freq['words'].str.contains(exclude)\n",
    "\n",
    "# freq[(filter3) & ~(filter4)].head(10)\n",
    "# freq[filter5]\n",
    "\n",
    "top10_tech_df = freq[(filter3) & ~(filter4)].head(10)\n",
    "# make the names nicer\n",
    "top10_tech_df['words'] = top10_tech_df['words'].str.replace('_',' ').str.strip().str.upper()\n",
    "top10_tech_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>words</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>AZURE</td>\n",
       "      <td>539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>AWS</td>\n",
       "      <td>348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429</th>\n",
       "      <td>SQL</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538</th>\n",
       "      <td>PYTHON</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920</th>\n",
       "      <td>TABLEAU</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1394</th>\n",
       "      <td>GCP</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1571</th>\n",
       "      <td>EXCEL</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1631</th>\n",
       "      <td>HADOOP</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1720</th>\n",
       "      <td>NO SQL</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1910</th>\n",
       "      <td>POWER BI</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         words  count\n",
       "49       AZURE    539\n",
       "88         AWS    348\n",
       "429        SQL    102\n",
       "538     PYTHON     85\n",
       "920    TABLEAU     46\n",
       "1394       GCP     27\n",
       "1571     EXCEL     23\n",
       "1631    HADOOP     22\n",
       "1720    NO SQL     20\n",
       "1910  POWER BI     17"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# freq is a word (single word) count\n",
    "# to get phrases, replace powerbi and power bi with power_bi\n",
    "tech1 = ['sql','python','excel',\n",
    "        'tableau','aws','gcp',\n",
    "        'azure','qlik','matlab','hadoop',\n",
    "        'looker','domo','dbt', 'kafka','spark'\n",
    "        'datorama','_r_']\n",
    "tech2 = tech1 + (list(set(multi_tech.values())))\n",
    "tech = '|'.join(tech2)\n",
    "# these words contain things like aws and excel\n",
    "exclude1 = ['capital','laws','excellen']\n",
    "exclude = '|'.join(exclude1)\n",
    "\n",
    "filter3 = freq['words'].str.contains(tech)\n",
    "filter4 = freq['words'].str.contains(exclude)\n",
    "\n",
    "# freq[(filter3) & ~(filter4)].head(10)\n",
    "# freq[filter5]\n",
    "\n",
    "top10_tech_df = freq[(filter3) & ~(filter4)].head(10)\n",
    "# make the names nicer\n",
    "top10_tech_df['words'] = top10_tech_df['words'].str.replace('_',' ').str.strip().str.upper()\n",
    "top10_tech_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Writing the data to our database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "# looks like we need none of the above, the auth is contained inside pygsheets\n",
    "# need to enable google sheets api and google drive api \n",
    "# then share your spreadsheet with the client_email in your key.json file\n",
    "import pygsheets\n",
    "client = pygsheets.authorize(service_file='jason2021-key.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Pvolo - Bag1', 'Data Analyst Course planning', 'Linkedin Learning Project']"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trying one from jason@wvc, also one from jasonsan to share\n",
    "# so it doesn't matter who the owner of the google sheet is, only that we share it with the client_email\n",
    "client.spreadsheet_titles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open spreadsheet name\n",
    "spreadsheet1 = client.open(\"Linkedin Learning Project\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Worksheet 'Raw Data' index:0>,\n",
       " <Worksheet 'Top 10 Skills' index:1>,\n",
       " <Worksheet 'Salaries' index:2>]"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the names of the tabs or worksheets\n",
    "spreadsheet1.worksheets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open the worksheet or tab\n",
    "worksheet1 = spreadsheet1.worksheet_by_title('Raw Data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Job Location</th>\n",
       "      <th>Job URL</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Seniority Level</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Job Function</th>\n",
       "      <th>Industries</th>\n",
       "      <th>Min_Salary</th>\n",
       "      <th>Max_Salary</th>\n",
       "      <th>Avg_Salary</th>\n",
       "      <th>Min_Salary_Num</th>\n",
       "      <th>Max_Salary_Num</th>\n",
       "      <th>Avg_Salary_Num</th>\n",
       "      <th>City</th>\n",
       "      <th>State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Analyst, Research and Insights</td>\n",
       "      <td>Google</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2616405818/...</td>\n",
       "      <td>Note: By applying to this position you will h...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Information Services and Internet</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tableau Business Intelligence Senior Analyst</td>\n",
       "      <td>Salesforce</td>\n",
       "      <td>San Francisco, CA</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2615511362/...</td>\n",
       "      <td>To get the best candidate experience, please ...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Research, Analyst, and Information Technology</td>\n",
       "      <td>Computer Software, Information Technology and ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>CA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Analyst, Churn Analytics</td>\n",
       "      <td>Disney Streaming Services</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2620559153/...</td>\n",
       "      <td>Job Summary: We are looking for an experience...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Quality Assurance, Information Technology, and...</td>\n",
       "      <td>Marketing and Advertising, Computer Software, ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Product Data Analyst II</td>\n",
       "      <td>Uber</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2632664783/...</td>\n",
       "      <td>About The RoleAre you interested in driving b...</td>\n",
       "      <td>Not Applicable</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Computer Software, Consumer Services, and Info...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Senior Data Analyst - Content</td>\n",
       "      <td>Hulu</td>\n",
       "      <td>Santa Monica, CA</td>\n",
       "      <td>https://www.linkedin.com/jobs/view/2620557390/...</td>\n",
       "      <td>SummaryThe Disney Median &amp; Entertainment Dist...</td>\n",
       "      <td>Mid-Senior level</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>Business Development, Analyst, and Engineering</td>\n",
       "      <td>Marketing and Advertising, Computer Software, ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Santa Monica</td>\n",
       "      <td>CA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Job Title                    Company  \\\n",
       "0           Data Analyst, Research and Insights                     Google   \n",
       "1  Tableau Business Intelligence Senior Analyst                 Salesforce   \n",
       "2                 Data Analyst, Churn Analytics  Disney Streaming Services   \n",
       "3                       Product Data Analyst II                       Uber   \n",
       "4                 Senior Data Analyst - Content                       Hulu   \n",
       "\n",
       "        Job Location                                            Job URL  \\\n",
       "0       New York, NY  https://www.linkedin.com/jobs/view/2616405818/...   \n",
       "1  San Francisco, CA  https://www.linkedin.com/jobs/view/2615511362/...   \n",
       "2       New York, NY  https://www.linkedin.com/jobs/view/2620559153/...   \n",
       "3       New York, NY  https://www.linkedin.com/jobs/view/2632664783/...   \n",
       "4   Santa Monica, CA  https://www.linkedin.com/jobs/view/2620557390/...   \n",
       "\n",
       "                                     Job Description   Seniority Level  \\\n",
       "0   Note: By applying to this position you will h...    Not Applicable   \n",
       "1   To get the best candidate experience, please ...    Not Applicable   \n",
       "2   Job Summary: We are looking for an experience...  Mid-Senior level   \n",
       "3   About The RoleAre you interested in driving b...    Not Applicable   \n",
       "4   SummaryThe Disney Median & Entertainment Dist...  Mid-Senior level   \n",
       "\n",
       "  Employment Type                                       Job Function  \\\n",
       "0       Full-time                             Information Technology   \n",
       "1       Full-time      Research, Analyst, and Information Technology   \n",
       "2       Full-time  Quality Assurance, Information Technology, and...   \n",
       "3       Full-time                             Information Technology   \n",
       "4       Full-time     Business Development, Analyst, and Engineering   \n",
       "\n",
       "                                          Industries Min_Salary Max_Salary  \\\n",
       "0                  Information Services and Internet                         \n",
       "1  Computer Software, Information Technology and ...                         \n",
       "2  Marketing and Advertising, Computer Software, ...                         \n",
       "3  Computer Software, Consumer Services, and Info...                         \n",
       "4  Marketing and Advertising, Computer Software, ...                         \n",
       "\n",
       "  Avg_Salary Min_Salary_Num Max_Salary_Num Avg_Salary_Num           City State  \n",
       "0                                                               New York    NY  \n",
       "1                                                          San Francisco    CA  \n",
       "2                                                               New York    NY  \n",
       "3                                                               New York    NY  \n",
       "4                                                           Santa Monica    CA  "
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Put sheet into a dataframe\n",
    "df = worksheet1.get_as_df()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in the tab write our dataframe with a starting coordinates row,col\n",
    "worksheet1.set_dataframe(jobs_df_withsal,start=(1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next the worksheet or tab\n",
    "worksheet2 = spreadsheet1.worksheet_by_title('Top 10 Skills')\n",
    "worksheet2.set_dataframe(top10_tech_df,start=(1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# next the worksheet or tab\n",
    "worksheet3 = spreadsheet1.worksheet_by_title('Salaries')\n",
    "worksheet3.set_dataframe(jobs_sal_df,start=(1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}