spider_02.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 上次课作业简单讲解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 通用ua\n",
    "ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 天气预报数据抓取\n",
    "# 入口页面：http://www.weather.com.cn/weather/101020100.shtml\n",
    "# 抓取最近7天天气预报信息\n",
    "# 进一步思考：如何从百度搜索入口进入然后获得天气预报信息？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 17火商品数据抓取\n",
    "# 入口页面：http://www.17huo.com/list-man-0-50011123-0--2m0.html'\n",
    "# 抓取商品的url、名称、代码和价格。\n",
    "# 难点：如何处理分页问题。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 使用并发提高抓取效率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 多线程\n",
    "# 充分利用CPU的数量并发工作。\n",
    "# Python多线程对计算密集型不友好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "thread MainThread is running...\n",
      "thread LoopThread is running...\n",
      "thread LoopThread >>> 1\n",
      "thread LoopThread >>> 2\n",
      "thread LoopThread >>> 3\n",
      "thread LoopThread >>> 4\n",
      "thread LoopThread >>> 5\n",
      "thread LoopThread ended.thread MainThread ended.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import time, threading\n",
    "\n",
    "# 新线程执行的代码:\n",
    "def loop():\n",
    "    print('thread %s is running...' % threading.current_thread().name)\n",
    "    n = 0\n",
    "    while n < 5:\n",
    "        n = n + 1\n",
    "        print('thread %s >>> %s' % (threading.current_thread().name, n))\n",
    "        time.sleep(1)\n",
    "    print('thread %s ended.' % threading.current_thread().name)\n",
    "\n",
    "print('thread %s is running...' % threading.current_thread().name)\n",
    "t = threading.Thread(target=loop, name='LoopThread')\n",
    "t.start()\n",
    "t.join()\n",
    "print('thread %s ended.' % threading.current_thread().name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2)(3, 4) {'four': 4, 'three': 3}\n",
      " {'one': 1, 'two': 2}\n"
     ]
    }
   ],
   "source": [
    "# 线程池\n",
    "# 直接使用线程存在管理难点，比如100个任务 vs 4核CPU\n",
    "import os\n",
    "import threadpool\n",
    "\n",
    "def tfunc(*args, **kws):\n",
    "    print(args, kws)\n",
    "    \n",
    "fargs_1 = ([1, 2], {'one': 1, 'two': 2})\n",
    "fargs_2 = ([3, 4], {'three': 3, 'four': 4})\n",
    "fargs = [fargs_1, fargs_2]\n",
    "pool = threadpool.ThreadPool(os.cpu_count())\n",
    "threads = threadpool.makeRequests(tfunc, fargs) \n",
    "[pool.putRequest(thread) for thread in threads]\n",
    "pool.wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "步长制药: 49.950, 49.810, 50.070, 49.720, \n",
      "顺丰控股: 49.900, 50.300, 50.450, 49.820, \n",
      "大同煤业: 6.900, 6.970, 7.070, 6.750, \n",
      "科大讯飞: 59.810, 60.100, 60.800, 59.510, \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "300ETF: 4.691, 4.658, 4.697, 4.658, \n",
      "\n",
      "中科信息: 56.160, 54.820, 56.450, 54.100, \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 利用线程池获取新浪股票接口数据\n",
    "import requests\n",
    "\n",
    "def get_stock_info(code):\n",
    "    api = 'http://hq.sinajs.cn/list='\n",
    "    r = requests.get(api + code)\n",
    "    data_str = r.text.split('=')[1].replace('\"', '')[:-2]\n",
    "    infos = data_str.split(',')\n",
    "    s = infos[0] + ': '\n",
    "    s += infos[1] + ', ' # 开\n",
    "    s += infos[3] + ', ' # 收\n",
    "    s += infos[4] + ', ' # 高\n",
    "    s += infos[5] + ', ' # 低\n",
    "    s += '\\n'\n",
    "    print(s)\n",
    "    \n",
    "codes = ['sh601001', 'sz002230', 'sh603858', 'sz002352', 'sz159919', 'sz300678']\n",
    "pool = threadpool.ThreadPool(os.cpu_count())\n",
    "threads = threadpool.makeRequests(get_stock_info, codes) \n",
    "[pool.putRequest(thread) for thread in threads]\n",
    "pool.wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 异步及其应用\n",
    "# 异步示意图：https://my.oschina.net/andylucc/blog/906327\n",
    "# 简单说就是请求发送之后就不管了，该忙什么忙什么，注册一个回调函数，请求完成返回数据之后自动调用这个函数进行处理。\n",
    "# 优点是在等待数据返回时，本来是不消耗CPU的，但还是等占着CPU资源，因为要等请求返回。现在直接忙别的去了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named 'requests_threads'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-1bde1f4e5385>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 使用异步线程抓取新浪股票接口数据\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mrequests_threads\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAsyncSession\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfetch_stock_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;34masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfetch_stock_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named 'requests_threads'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# 使用异步线程抓取新浪股票接口数据\n",
    "from requests_threads import AsyncSession\n",
    "\n",
    "def fetch_stock_info(session, codes):\n",
    "    async def fetch_stock_info():\n",
    "        responses = []\n",
    "        for code in codes:\n",
    "            responses.append(await session.get( 'http://hq.sinajs.cn/list=' + code))\n",
    "        for r in responses:\n",
    "            data_str = r.text.split('=')[1].replace('\"', '')[:-2]\n",
    "            infos = data_str.split(',')\n",
    "            s = infos[0] + ': '\n",
    "            s += infos[1] + ', ' # 开\n",
    "            s += infos[3] + ', ' # 收\n",
    "            s += infos[4] + ', ' # 高\n",
    "            s += infos[5] + ', ' # 低\n",
    "            s += '\\n'\n",
    "            print(s)\n",
    "    return fetch_stock_info\n",
    "\n",
    "session = AsyncSession(n=os.cpu_count())\n",
    "codes = ['sh601001', 'sz002230', 'sh603858', 'sz002352', 'sz159919', 'sz300678']\n",
    "f = fetch_stock_info(session, codes)\n",
    "session.run(f) # 执行完一次后想要再次执行需要重启notebook，这个是twisted库的坑，暂时懒得解决了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 作业\n",
    "\n",
    "# 1. 利用www.okex.com提供的API获取币币（vs BTC）的行情信息\n",
    "#    币列表参考：https://github.com/yingl/bithelper/blob/master/okex_data/coins.py\n",
    "#    API参考：https://www.okex.com/rest_api.html\n",
    "# 2. 抓取链家商品房数据，上海二手房前3页，每条记录至少抓取3个字段。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}