Cha 8 -反爬虫问题.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8.3如何“反反爬虫”？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.3.1修改请求 header"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'User-Agent': 'python-requests/2.12.4', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "r = requests.get('http://www.santostang.com')\n",
    "print (r.request.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "\n",
    "link = 'http://www.santostang.com'\n",
    "headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "r = requests.get(link, headers= headers)\n",
    "print (r.request.headers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.3.2 修改爬虫的间隔时间"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0001144409179688\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "t1 = time.time()\n",
    "time.sleep(2)\n",
    "t2 = time.time()\n",
    "total_time = t2-t1\n",
    "print (total_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3481693303048349\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import random\n",
    "\n",
    "sleep_time = random.randint(0,2) + random.random()\n",
    "print (sleep_time)\n",
    "time.sleep(sleep_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/08/hello-python/\n",
      "这篇博客的标题为:  Hello Python!\n",
      "开始休息:  0.16292490492777212 秒\n",
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/07/echarts%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b02-%e5%8d%95%e9%a1%b5%e9%9d%a2%e5%a4%9a%e5%bc%a0%e5%9b%be%e8%a1%a8/\n",
      "这篇博客的标题为:  echarts学习笔记(2) — 同一页面多图表\n",
      "开始休息:  1.912631031656519 秒\n",
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/07/echarts%e5%ad%a6%e4%b9%a0%e7%ac%94%e8%ae%b01-%e4%bd%bf%e7%94%a8%e6%a8%a1%e5%9d%97%e5%8c%96%e5%8d%95%e6%96%87%e4%bb%b6%e5%bc%95%e5%85%a5/\n",
      "这篇博客的标题为:  echarts学习笔记(1) — 模块化单文件引入\n",
      "开始休息:  1.3634313119416182 秒\n",
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/06/%e3%80%90%e7%88%ac%e8%99%ab%e4%ba%8c%e3%80%91%e7%88%ac%e8%99%ab%e7%9a%84%e6%a1%86%e6%9e%b6%e5%92%8c%e5%9f%ba%e6%9c%ac%e8%ae%ae%e9%a2%98/\n",
      "这篇博客的标题为:  【爬虫二】爬虫的框架和基本议题\n",
      "开始休息:  2.0205314818737516 秒\n",
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/06/%e3%80%90%e7%88%ac%e8%99%ab%e4%b8%80%e3%80%91%e6%9c%80%e7%ae%80%e5%8d%95%e7%9a%84%e7%88%ac%e8%99%ab%ef%bc%8c%e9%9b%b6%e5%9f%ba%e7%a1%80%e6%95%99%e5%ad%a6/\n",
      "这篇博客的标题为:  【爬虫一】最简单的爬虫，零基础教学\n",
      "开始休息:  2.446761436097069 秒\n",
      "开始爬取这篇博客:  http://www.santostang.com/2017/03/02/hello-world/\n",
      "这篇博客的标题为:  Hello world!\n",
      "开始休息:  0.8005131789714476 秒\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "import random\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "\n",
    "def scrap(link):\n",
    "    headers = {'User-Agent' : 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'} \n",
    "    r = requests.get(link, headers= headers)\n",
    "    html = r.text\n",
    "    soup = BeautifulSoup(html, \"lxml\")\n",
    "    return soup\n",
    "\n",
    "soup = scrap(link)\n",
    "title_list = soup.find_all(\"h1\", class_=\"post-title\")\n",
    "for eachone in title_list:\n",
    "    url = eachone.a['href']\n",
    "    print ('开始爬取这篇博客: ', url)\n",
    "    soup_article = scrap(url)\n",
    "    title = soup_article.find(\"h1\", class_=\"view-title\").text.strip()\n",
    "    print ('这篇博客的标题为: ', title)\n",
    "    sleep_time = random.randint(0,2) + random.random()\n",
    "    print ('开始休息: ', sleep_time, '秒')\n",
    "    time.sleep(sleep_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scrap_times = 0\n",
    "for eachone in title_list:\n",
    "    url = eachone.a['href']\n",
    "    print ('开始爬取这篇博客: ', url)\n",
    "    soup_article = scrap(url)\n",
    "    title = soup_article.find(\"h1\", class_=\"view-title\").text.strip()\n",
    "    print ('这篇博客的标题为: ', title)\n",
    "    \n",
    "    scrap_times += 1\n",
    "    if scrap_times % 5 == 0:\n",
    "        sleep_time = 10 + random.random()\n",
    "    else:\n",
    "        sleep_time = random.randint(0,2) + random.random()\n",
    "    time.sleep(sleep_time)\n",
    "    print ('开始休息: ', sleep_time, '秒')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8.3.3 使用代理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "link = \"http://www.santostang.com/\"\n",
    "proxies = {'http':'http://xxx.xxx.xxx.xxx:xxxx'}\n",
    "response = requests.get(link, proxies=proxies)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}