scrape_swiss_brands.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2 \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%load_ext soup\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from os import path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = \"http://www.brandsoftheworld.com/logos/countries/ch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def retrieve_swiss_brands():\n",
    "    arguments = {'page':0} # grab HTML data\n",
    "    all_brands = []\n",
    "    # retrieve the html web page containing the list of brands\n",
    "    r = requests.get(url, params=arguments)\n",
    "    soup_of_brands = BeautifulSoup(r.text, 'html.parser')\n",
    "    # select the tags containing the names of these brands\n",
    "    resulting_brands = soup_of_brands.findAll(\"span\", { \"class\" : \"title\" })\n",
    "    # remove the tags and only keep the actual brands\n",
    "    brands = [brand.string for brand in resulting_brands]\n",
    "    while(len(brands) > 0):\n",
    "        # print progress\n",
    "        print(\"Finished page \"+str(arguments['page'])+\" of swiss brands\")\n",
    "        all_brands += brands\n",
    "        arguments['page'] += 1\n",
    "        # rinse and repeat\n",
    "        r = requests.get(url, params=arguments)\n",
    "        soup_of_brands = BeautifulSoup(r.text, 'html.parser')\n",
    "        resulting_brands = soup_of_brands.findAll(\"span\", { \"class\" : \"title\" })\n",
    "        brands = [brand.string for brand in resulting_brands]\n",
    "    return all_brands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished page 0 of swiss brands\n",
      "Finished page 1 of swiss brands\n",
      "Finished page 2 of swiss brands\n",
      "Finished page 3 of swiss brands\n",
      "Finished page 4 of swiss brands\n",
      "Finished page 5 of swiss brands\n",
      "Finished page 6 of swiss brands\n",
      "Finished page 7 of swiss brands\n",
      "Finished page 8 of swiss brands\n",
      "Finished page 9 of swiss brands\n",
      "Finished page 10 of swiss brands\n",
      "Finished page 11 of swiss brands\n",
      "Finished page 12 of swiss brands\n",
      "Finished page 13 of swiss brands\n",
      "Finished page 14 of swiss brands\n",
      "Finished page 15 of swiss brands\n",
      "Finished page 16 of swiss brands\n",
      "Finished page 17 of swiss brands\n",
      "Finished page 18 of swiss brands\n",
      "Finished page 19 of swiss brands\n",
      "Finished page 20 of swiss brands\n",
      "Finished page 21 of swiss brands\n",
      "Finished page 22 of swiss brands\n",
      "Finished page 23 of swiss brands\n",
      "Finished page 24 of swiss brands\n",
      "Finished page 25 of swiss brands\n",
      "Finished page 26 of swiss brands\n",
      "Finished page 27 of swiss brands\n",
      "Finished page 28 of swiss brands\n",
      "Finished page 29 of swiss brands\n",
      "Finished page 30 of swiss brands\n",
      "Finished page 31 of swiss brands\n",
      "Finished page 32 of swiss brands\n",
      "Finished page 33 of swiss brands\n",
      "Finished page 34 of swiss brands\n",
      "Finished page 35 of swiss brands\n",
      "Finished page 36 of swiss brands\n",
      "Finished page 37 of swiss brands\n",
      "Finished page 38 of swiss brands\n",
      "Finished page 39 of swiss brands\n",
      "Finished page 40 of swiss brands\n",
      "Finished page 41 of swiss brands\n",
      "Finished page 42 of swiss brands\n",
      "Finished page 43 of swiss brands\n",
      "Finished page 44 of swiss brands\n",
      "Finished page 45 of swiss brands\n",
      "Finished page 46 of swiss brands\n",
      "Finished page 47 of swiss brands\n",
      "Finished page 48 of swiss brands\n",
      "Finished page 49 of swiss brands\n",
      "Finished page 50 of swiss brands\n",
      "Finished page 51 of swiss brands\n",
      "Finished page 52 of swiss brands\n",
      "Finished page 53 of swiss brands\n",
      "Finished page 54 of swiss brands\n",
      "Finished page 55 of swiss brands\n",
      "Finished page 56 of swiss brands\n",
      "Finished page 57 of swiss brands\n",
      "Finished page 58 of swiss brands\n"
     ]
    }
   ],
   "source": [
    "all_swiss_brands = retrieve_swiss_brands()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1885"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_swiss_brands)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Dump the list of contents into a file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('data/all_swiss_brands.pickle', 'wb') as fp:\n",
    "    pickle.dump(all_swiss_brands, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the file into a list of brands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open ('data/all_swiss_brands.pickle', 'rb') as fp:\n",
    "    brand_list = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1885"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(brand_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}