scraping_analytics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MVP - Blog level analytics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import bonobo\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from dateutil import parser\n",
    "from textstat.textstat import textstat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',\n",
    "    'referrer': 'https://google.com'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categories = ['post-production', 'color-correction', 'business', 'workflow', 'behind-the-scenes', 'production', 'announcement']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "articles_store = []\n",
    "\n",
    "def parse_category(url):\n",
    "    r = requests.get(url, headers=headers)\n",
    "    html = r.text.strip()\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    article_cards = soup.findAll(class_='post-content')\n",
    "\n",
    "    for article in article_cards:\n",
    "        title = article.find(class_='post-meta-title')\n",
    "        link = title.contents[0]['href']\n",
    "        print('Parsing URL:', link)\n",
    "        page = parse_page(link)\n",
    "        articles_store.append(page)\n",
    "        \n",
    "    next_link = find_next_link(soup)\n",
    "    \n",
    "    if next_link is not None:\n",
    "        print('Next page:', next_link)\n",
    "        parse_category(next_link)\n",
    "        \n",
    "    return None\n",
    "\n",
    "def find_next_link(soup_item):\n",
    "    bottom_nav = soup_item.find(class_='navigation')\n",
    "    \n",
    "    if bottom_nav == None:\n",
    "        return None\n",
    "    \n",
    "    links = bottom_nav.findAll('a')\n",
    "    next_page = links[-1]\n",
    "\n",
    "    if next_page.contents[0] == 'Next':\n",
    "        next_link = next_page['href']\n",
    "        return next_link\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_page(url):\n",
    "    r = requests.get(url, headers=headers)\n",
    "    html = r.text.strip()\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    # Header Content\n",
    "    header = soup.find(class_='entry-header')\n",
    "    read_time = extract_read_time(header)\n",
    "    title = extract_title(header)\n",
    "\n",
    "    author = extract_author(header)\n",
    "    categories = extract_categories(header)\n",
    "\n",
    "    date = extract_date(header)\n",
    "    dt = parser.parse(date)\n",
    "    month = dt.strftime(\"%B\")\n",
    "    weekday = dt.strftime(\"%A\")\n",
    "    \n",
    "    # Body Content\n",
    "    content = soup.find(class_='entry-content')\n",
    "    word_count = len(content.text.split())\n",
    "    reading_level = textstat.flesch_kincaid_grade(content.text)\n",
    "\n",
    "    links = content.find_all(\"a\")\n",
    "    link_count = len(links)\n",
    "\n",
    "    images = content.find_all(\"img\")\n",
    "    image_count = len(images)\n",
    "    \n",
    "    page_data = {\n",
    "        'reading_time' : read_time,\n",
    "        'title': title,\n",
    "        'date': date,\n",
    "        'month': month,\n",
    "        'weekday': weekday,\n",
    "        'author': author,\n",
    "        'categories': categories,\n",
    "        'word_count': word_count,\n",
    "        'reading_level': reading_level,\n",
    "        'link_count': link_count,\n",
    "        'image_count': image_count\n",
    "    }\n",
    "    \n",
    "    return page_data\n",
    "    \n",
    "def extract_read_time(header):\n",
    "    html_str = header.find(class_='read-time')\n",
    "    time_str = html_str.contents[0].strip().lower().split()[0]\n",
    "    time_int = int(time_str)\n",
    "    return time_int\n",
    "\n",
    "def extract_title(header):\n",
    "    html_str = header.find(class_='post-meta-title')\n",
    "    title_str = html_str.contents[0].strip()\n",
    "    return title_str\n",
    "\n",
    "def extract_date(header):\n",
    "    html_str = header.find(class_='single-post-date')\n",
    "    date_str = html_str.contents[0].strip()\n",
    "    return date_str\n",
    "\n",
    "def extract_author(header):\n",
    "    html_str = header.find(class_='author-name')\n",
    "    author_str = html_str.find('a').contents[0].strip()\n",
    "    return author_str\n",
    "\n",
    "def extract_categories(header):\n",
    "    html_str = header.find(class_='single-post-cat')\n",
    "    categories = html_str.findAll('a')\n",
    "    cat_names = []\n",
    "    for cat_link in categories:\n",
    "        cat_name = cat_link.contents[0].strip().lower()\n",
    "        cat_names.append(cat_name)\n",
    "    return cat_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for category in categories:\n",
    "    url = 'https://blog.frame.io/category/' + category + '/'\n",
    "    print('Parsing category', category)\n",
    "    parse_category(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(articles_store)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "articles_store[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(articles_store, open(\"articles.p\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('articles.json', 'w') as f:\n",
    "    json.dump(articles_store, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary stats to JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Average reading time\n",
    "- Count of posts by category\n",
    "- Count of posts by author\n",
    "- Posts by month\n",
    "- Posts by day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Summary Statistics\n",
    "times = []\n",
    "months = []\n",
    "weekdays = []\n",
    "authors = []\n",
    "categories = []\n",
    "\n",
    "for article in articles_store:\n",
    "    # Average Reading Time\n",
    "    times.append(article['reading_time'])\n",
    "    average_time = sum(times) / float(len(times))\n",
    "    average_time = round(average_time, 2)\n",
    "    \n",
    "    # Posts by Month\n",
    "    months.append(article['month'])\n",
    "    month_count = Counter(months)\n",
    "    \n",
    "    # Posts by Weekday\n",
    "    weekdays.append(article['weekday'])\n",
    "    weekday_count = Counter(weekdays)\n",
    "    \n",
    "    # Count by Category\n",
    "    categories += article['categories']\n",
    "    category_count = Counter(categories)\n",
    "    \n",
    "    # Count by Author\n",
    "    authors.append(article['author'])\n",
    "    author_count = Counter(authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Average reading time:\", average_time)\n",
    "print(\"Posts by month\", month_count)\n",
    "print(\"Posts by weekday\", weekday_count)\n",
    "print(\"Posts by category\", category_count)\n",
    "print(\"Posts by author\", author_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "stats = { \n",
    "    'reading_time': average_time, \n",
    "    'num_articles': len(articles_store) \n",
    "}\n",
    "\n",
    "with open('data/stats.json', 'w') as f:\n",
    "    json.dump(stats, f)\n",
    "\n",
    "with open('data/weekday.json', 'w') as f:\n",
    "    json.dump(weekday_count, f)\n",
    "    \n",
    "with open('data/month.json', 'w') as f:\n",
    "    json.dump(month_count, f)\n",
    "    \n",
    "with open('data/category.json', 'w') as f:\n",
    "    json.dump(category_count, f)\n",
    "\n",
    "with open('data/author.json', 'w') as f:\n",
    "    json.dump(author_count, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}