spider/慕课课件下载.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-02T12:51:15.439691Z",
     "start_time": "2020-04-02T12:51:15.436872Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from selenium import webdriver\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-02T12:48:15.750997Z",
     "start_time": "2020-04-02T12:48:15.748665Z"
    }
   },
   "outputs": [],
   "source": [
    "dir_path = '电工学/'\n",
    "url = 'http://www.icourse163.org/learn/HIT-1001998009?tid=1002098003#/learn/content'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-02T13:01:23.895292Z",
     "start_time": "2020-04-02T13:01:21.674177Z"
    }
   },
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-02T13:41:01.478542Z",
     "start_time": "2020-04-02T13:36:20.629957Z"
    }
   },
   "outputs": [],
   "source": [
    "driver.get(url) # 打开网页​\n",
    "last_file = ''\n",
    "jump = 5\n",
    "for i, ele in enumerate(driver.find_elements_by_class_name('j-titleName')[jump:]):\n",
    "    driver.get(url)\n",
    "    time.sleep(1) # 等待页面加载完成\n",
    "    driver.find_elements_by_class_name('j-titleName')[0].click()\n",
    "    driver.find_elements_by_class_name('j-titleName')[i+jump].click()\n",
    "    html = bs(driver.page_source, 'lxml')  \n",
    "    # driver.get('http://www.baidu.com')\n",
    "    driver.find_elements_by_class_name('j-titleName')[0].click()\n",
    "    for j, ll in enumerate(html.select('.u-learnLesson.normal.f-cb.f-pr')):\n",
    "        print(i, j)\n",
    "        driver.get(url)\n",
    "        time.sleep(1)\n",
    "        driver.find_elements_by_class_name('j-titleName')[0].click()\n",
    "        driver.find_elements_by_class_name('j-titleName')[i+jump].click()\n",
    "#         driver.find_elements_by_class_name('lsicon')[j].click()\n",
    "        html = bs(driver.page_source, 'lxml')  \n",
    "        driver.find_element_by_id(html.select('.u-learnLesson.normal.f-cb.f-pr')[j]['id']).click()\n",
    "        time.sleep(1)\n",
    "        html = bs(driver.page_source, 'lxml')     \n",
    "        file_path = dir_path + html.select('.up.j-up.f-thide')[1].text.replace(' ', '') + '.pdf'\n",
    "        if last_file == file_path:\n",
    "            print('jump', j)\n",
    "            continue\n",
    "#         if len(html.select('li.f-fl')) == 1:\n",
    "#             continue\n",
    "        for fl in html.select('li.f-fl'):\n",
    "            if '文档' in fl['title']:\n",
    "                id_ = fl['id']\n",
    "                break\n",
    "            \n",
    "        driver.find_element_by_id(id_).click()\n",
    "        time.sleep(1)\n",
    "        html = bs(driver.page_source, 'lxml')  \n",
    "        print(file_path)\n",
    "        r = requests.get(html.select('.j-downpdf')[0]['href'])\n",
    "        with open(file_path,\"wb\") as code:\n",
    "            code.write(r.content)\n",
    "        last_file = file_path"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}