Scattertext.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting scattertext\n",
      "  Downloading scattertext-0.1.0.0-py3-none-any.whl (7.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 7.3 MB 14.2 MB/s eta 0:00:01     |██████                          | 1.4 MB 14.2 MB/s eta 0:00:01��████████████████████▌         | 5.1 MB 14.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting statsmodels\n",
      "  Downloading statsmodels-0.12.2-cp36-cp36m-manylinux1_x86_64.whl (9.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.5 MB 38.0 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting mock\n",
      "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
      "Requirement already satisfied: scikit-learn in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scattertext) (0.24.0)\n",
      "Requirement already satisfied: numpy in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scattertext) (1.19.5)\n",
      "Collecting flashtext\n",
      "  Downloading flashtext-2.7.tar.gz (14 kB)\n",
      "Requirement already satisfied: scipy in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scattertext) (1.5.3)\n",
      "Requirement already satisfied: pandas in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scattertext) (1.1.5)\n",
      "Requirement already satisfied: six in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scattertext) (1.15.0)\n",
      "Collecting patsy>=0.5\n",
      "  Downloading patsy-0.5.1-py2.py3-none-any.whl (231 kB)\n",
      "\u001b[K     |████████████████████████████████| 231 kB 47.6 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: threadpoolctl>=2.0.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scikit-learn->scattertext) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from scikit-learn->scattertext) (1.0.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from pandas->scattertext) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from pandas->scattertext) (2020.5)\n",
      "Building wheels for collected packages: flashtext\n",
      "  Building wheel for flashtext (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for flashtext: filename=flashtext-2.7-py2.py3-none-any.whl size=9299 sha256=f1a1dbca2ea906393b9ba04943de383f06dc1df9345e67508900f0fc46987391\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/a6/2c/3e/f78da03b67d65cc01f2d8dcf0b53d00f683eac36d03027be42\n",
      "Successfully built flashtext\n",
      "Installing collected packages: patsy, statsmodels, mock, flashtext, scattertext\n",
      "Successfully installed flashtext-2.7 mock-4.0.3 patsy-0.5.1 scattertext-0.1.0.0 statsmodels-0.12.2\n",
      "Collecting spacy\n",
      "  Downloading spacy-3.0.3-cp36-cp36m-manylinux2014_x86_64.whl (12.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 12.8 MB 10.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting tqdm<5.0.0,>=4.38.0\n",
      "  Downloading tqdm-4.56.2-py2.py3-none-any.whl (72 kB)\n",
      "\u001b[K     |████████████████████████████████| 72 kB 14 kB/s s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (20.4)\n",
      "Collecting wasabi<1.1.0,>=0.8.1\n",
      "  Downloading wasabi-0.8.2-py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (2.24.0)\n",
      "Collecting pydantic<1.8.0,>=1.7.1\n",
      "  Downloading pydantic-1.7.3-cp36-cp36m-manylinux2014_x86_64.whl (9.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.2 MB 58.1 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: jinja2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (2.11.2)\n",
      "Collecting srsly<3.0.0,>=2.4.0\n",
      "  Downloading srsly-2.4.0-cp36-cp36m-manylinux2014_x86_64.whl (456 kB)\n",
      "\u001b[K     |████████████████████████████████| 456 kB 51.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: setuptools in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (49.2.0.post20200712)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (1.7.0)\n",
      "Collecting catalogue<2.1.0,>=2.0.1\n",
      "  Downloading catalogue-2.0.1-py3-none-any.whl (9.6 kB)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy) (1.19.5)\n",
      "Collecting spacy-legacy<3.1.0,>=3.0.0\n",
      "  Downloading spacy_legacy-3.0.1-py2.py3-none-any.whl (7.0 kB)\n",
      "Collecting preshed<3.1.0,>=3.0.2\n",
      "  Downloading preshed-3.0.5-cp36-cp36m-manylinux2014_x86_64.whl (126 kB)\n",
      "\u001b[K     |████████████████████████████████| 126 kB 25.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting cymem<2.1.0,>=2.0.2\n",
      "  Downloading cymem-2.0.5-cp36-cp36m-manylinux2014_x86_64.whl (35 kB)\n",
      "Collecting pathy\n",
      "  Downloading pathy-0.4.0-py3-none-any.whl (36 kB)\n",
      "Collecting typer<0.4.0,>=0.3.0\n",
      "  Downloading typer-0.3.2-py3-none-any.whl (21 kB)\n",
      "Collecting typing-extensions>=3.7.4; python_version < \"3.8\"\n",
      "  Downloading typing_extensions-3.7.4.3-py3-none-any.whl (22 kB)\n",
      "Collecting thinc<8.1.0,>=8.0.0\n",
      "  Downloading thinc-8.0.1-cp36-cp36m-manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1 MB 35.2 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting murmurhash<1.1.0,>=0.28.0\n",
      "  Downloading murmurhash-1.0.5-cp36-cp36m-manylinux2014_x86_64.whl (20 kB)\n",
      "Collecting blis<0.8.0,>=0.4.0\n",
      "  Downloading blis-0.7.4-cp36-cp36m-manylinux2014_x86_64.whl (9.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.8 MB 38.1 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from packaging>=20.0->spacy) (2.4.7)\n",
      "Requirement already satisfied: six in /srv/conda/envs/notebook/lib/python3.6/site-packages (from packaging>=20.0->spacy) (1.15.0)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.25.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.6.20)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Collecting dataclasses>=0.6; python_version < \"3.7\"\n",
      "  Downloading dataclasses-0.8-py3-none-any.whl (19 kB)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from jinja2->spacy) (1.1.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->spacy) (3.1.0)\n",
      "Collecting smart-open<4.0.0,>=2.2.0\n",
      "  Downloading smart_open-3.0.0.tar.gz (113 kB)\n",
      "\u001b[K     |████████████████████████████████| 113 kB 44.1 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting click<7.2.0,>=7.1.1\n",
      "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
      "\u001b[K     |████████████████████████████████| 82 kB 999 kB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting contextvars<3,>=2.4; python_version < \"3.7\"\n",
      "  Downloading contextvars-2.4.tar.gz (9.6 kB)\n",
      "Collecting immutables>=0.9\n",
      "  Downloading immutables-0.15-cp36-cp36m-manylinux1_x86_64.whl (100 kB)\n",
      "\u001b[K     |████████████████████████████████| 100 kB 9.6 MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: smart-open, contextvars\n",
      "  Building wheel for smart-open (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for smart-open: filename=smart_open-3.0.0-py3-none-any.whl size=107097 sha256=adcfc3ced2c85e1e075e380f9faa9e7d18ef274b4843ecc8c2473f1420f5c1a5\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/88/2a/d4/f2e9023989d4d4b3574f268657cb6cd23994665a038803f547\n",
      "  Building wheel for contextvars (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for contextvars: filename=contextvars-2.4-py3-none-any.whl size=7664 sha256=3de68c32b386832960d49e784b0984242da723b14b5cd296018277710584a1d1\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/41/11/53/911724983aa48deb94792432e14e518447212dd6c5477d49d3\n",
      "Successfully built smart-open contextvars\n",
      "Installing collected packages: tqdm, wasabi, dataclasses, pydantic, catalogue, srsly, spacy-legacy, murmurhash, cymem, preshed, click, typer, smart-open, pathy, typing-extensions, immutables, contextvars, blis, thinc, spacy\n",
      "Successfully installed blis-0.7.4 catalogue-2.0.1 click-7.1.2 contextvars-2.4 cymem-2.0.5 dataclasses-0.8 immutables-0.15 murmurhash-1.0.5 pathy-0.4.0 preshed-3.0.5 pydantic-1.7.3 smart-open-3.0.0 spacy-3.0.3 spacy-legacy-3.0.1 srsly-2.4.0 thinc-8.0.1 tqdm-4.56.2 typer-0.3.2 typing-extensions-3.7.4.3 wasabi-0.8.2\r\n"
     ]
    }
   ],
   "source": [
    "!pip install scattertext\n",
    "!pip install spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting zh-core-web-sm==3.0.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.0.0/zh_core_web_sm-3.0.0-py3-none-any.whl (49.5 MB)\n",
      "\u001b[K     |████████████████████████████████| 49.5 MB 90 kB/s s eta 0:00:01     |██████████████▌                 | 22.5 MB 36.7 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting spacy-pkuseg<0.1.0,>=0.0.27\n",
      "  Downloading spacy_pkuseg-0.0.28-cp36-cp36m-manylinux2014_x86_64.whl (2.4 MB)\n",
      "\u001b[K     |████████████████████████████████| 2.4 MB 19.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: spacy<3.1.0,>=3.0.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from zh-core-web-sm==3.0.0) (3.0.3)\n",
      "Requirement already satisfied: numpy>=1.16.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy-pkuseg<0.1.0,>=0.0.27->zh-core-web-sm==3.0.0) (1.19.5)\n",
      "Requirement already satisfied: cython>=0.25 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy-pkuseg<0.1.0,>=0.0.27->zh-core-web-sm==3.0.0) (0.29.21)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.3.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy-pkuseg<0.1.0,>=0.0.27->zh-core-web-sm==3.0.0) (2.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4; python_version < \"3.8\" in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.7.4.3)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.7.4)\n",
      "Requirement already satisfied: pathy in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.4.0)\n",
      "Requirement already satisfied: jinja2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.11.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (20.4)\n",
      "Requirement already satisfied: setuptools in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (49.2.0.post20200712)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.7.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.24.0)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (8.0.1)\n",
      "Requirement already satisfied: pydantic<1.8.0,>=1.7.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.7.3)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.0.5)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.0.5)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.0.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.8.2)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (4.56.2)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.0.1)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.0.5)\n",
      "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.3.2)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6; python_version < \"3.7\" in /srv/conda/envs/notebook/lib/python3.6/site-packages (from pathy->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.8)\n",
      "Requirement already satisfied: smart-open<4.0.0,>=2.2.0 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from pathy->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.0.0)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from jinja2->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.1.1)\n",
      "Requirement already satisfied: six in /srv/conda/envs/notebook/lib/python3.6/site-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.15.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from packaging>=20.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.1.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (1.25.10)\n",
      "Requirement already satisfied: contextvars<3,>=2.4; python_version < \"3.7\" in /srv/conda/envs/notebook/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (2.4)\n",
      "Requirement already satisfied: click<7.2.0,>=7.1.1 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from typer<0.4.0,>=0.3.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (7.1.2)\n",
      "Requirement already satisfied: immutables>=0.9 in /srv/conda/envs/notebook/lib/python3.6/site-packages (from contextvars<3,>=2.4; python_version < \"3.7\"->thinc<8.1.0,>=8.0.0->spacy<3.1.0,>=3.0.0->zh-core-web-sm==3.0.0) (0.15)\n",
      "Installing collected packages: spacy-pkuseg, zh-core-web-sm\n",
      "Successfully installed spacy-pkuseg-0.0.28 zh-core-web-sm-3.0.0\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('zh_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download zh_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scattertext as st\n",
    "import pandas as pd\n",
    "import spacy\n",
    "from scattertext import SampleCorpora, PhraseMachinePhrases, dense_rank, RankDifference, AssociationCompactor, produce_scattertext_explorer\n",
    "from scattertext.CorpusFromPandas import CorpusFromPandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>text</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>十三五 第一篇 指导思想、主要目标和发展理念</td>\n",
       "      <td>“十三五”时期是全面建成小康社会决胜阶段。必须认真贯彻党中央战略决策和部署，准确把握国内外发...</td>\n",
       "      <td>十三五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>十三五 第二篇 实施创新驱动发展战略</td>\n",
       "      <td>把发展基点放在创新上，以科技创新为核心，以人才发展为支撑，推动科技创新与大众创业万众创新有机...</td>\n",
       "      <td>十三五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>十三五 第三篇 构建发展新体制</td>\n",
       "      <td>发挥经济体制改革牵引作用，正确处理政府和市场关系，在重点领域和关键环节改革上取得突破性进展，...</td>\n",
       "      <td>十三五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>十三五 第四篇 推进农业现代化</td>\n",
       "      <td>农业是全面建成小康社会和实现现代化的基础，必须加快转变农业发展方式，着力构建现代农业产业体系...</td>\n",
       "      <td>十三五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>十三五 第五篇 优化现代产业体系</td>\n",
       "      <td>围绕结构深度调整、振兴实体经济，推进供给侧结构性改革，培育壮大新兴产业，改造提升传统产业，加...</td>\n",
       "      <td>十三五</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  speaker                                               text  \\\n",
       "0  十三五 第一篇 指导思想、主要目标和发展理念  “十三五”时期是全面建成小康社会决胜阶段。必须认真贯彻党中央战略决策和部署，准确把握国内外发...   \n",
       "1      十三五 第二篇 实施创新驱动发展战略  把发展基点放在创新上，以科技创新为核心，以人才发展为支撑，推动科技创新与大众创业万众创新有机...   \n",
       "2         十三五 第三篇 构建发展新体制  发挥经济体制改革牵引作用，正确处理政府和市场关系，在重点领域和关键环节改革上取得突破性进展，...   \n",
       "3         十三五 第四篇 推进农业现代化  农业是全面建成小康社会和实现现代化的基础，必须加快转变农业发展方式，着力构建现代农业产业体系...   \n",
       "4        十三五 第五篇 优化现代产业体系  围绕结构深度调整、振兴实体经济，推进供给侧结构性改革，培育壮大新兴产业，改造提升传统产业，加...   \n",
       "\n",
       "  party  \n",
       "0   十三五  \n",
       "1   十三五  \n",
       "2   十三五  \n",
       "3   十三五  \n",
       "4   十三五  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('plan.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>text</th>\n",
       "      <th>party</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>十二五 第十二篇 互利共赢 提高对外开放水平</td>\n",
       "      <td>适应我国对外开放由出口和吸收外资为主转向进口和出口、吸收外资和对外投资并重的新形势，必须实行...</td>\n",
       "      <td>十二五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>十二五 第十三篇 发展民主 推进社会主义政治文明建设</td>\n",
       "      <td>坚持党的领导、人民当家作主、依法治国有机统一，发展社会主义民主政治，建设社会主义法治国家。\\...</td>\n",
       "      <td>十二五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>十二五 第十四篇 深化合作 建设中华民族共同家园</td>\n",
       "      <td>从中华民族根本利益出发，推进“一国两制”实践和祖国和平统一大业，深化内地与港澳经贸合作，推进...</td>\n",
       "      <td>十二五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>十二五 第十五篇 军民融合 加强国防和军队现代化建设</td>\n",
       "      <td>着眼国家安全和发展战略全局，统筹经济建设和国防建设，在全面建设小康社会进程中实现富国和强军的...</td>\n",
       "      <td>十二五</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>十二五 第十六篇 强化实施 实现宏伟发展蓝图</td>\n",
       "      <td>本规划经过全国人民代表大会审议批准，具有法律效力。要举全国之力，集全民之智，实现未来五年宏伟...</td>\n",
       "      <td>十二五</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       speaker  \\\n",
       "31      十二五 第十二篇 互利共赢 提高对外开放水平   \n",
       "32  十二五 第十三篇 发展民主 推进社会主义政治文明建设   \n",
       "33    十二五 第十四篇 深化合作 建设中华民族共同家园   \n",
       "34  十二五 第十五篇 军民融合 加强国防和军队现代化建设   \n",
       "35      十二五 第十六篇 强化实施 实现宏伟发展蓝图   \n",
       "\n",
       "                                                 text party  \n",
       "31  适应我国对外开放由出口和吸收外资为主转向进口和出口、吸收外资和对外投资并重的新形势，必须实行...   十二五  \n",
       "32  坚持党的领导、人民当家作主、依法治国有机统一，发展社会主义民主政治，建设社会主义法治国家。\\...   十二五  \n",
       "33  从中华民族根本利益出发，推进“一国两制”实践和祖国和平统一大业，深化内地与港澳经贸合作，推进...   十二五  \n",
       "34  着眼国家安全和发展战略全局，统筹经济建设和国防建设，在全面建设小康社会进程中实现富国和强军的...   十二五  \n",
       "35  本规划经过全国人民代表大会审议批准，具有法律效力。要举全国之力，集全民之智，实现未来五年宏伟...   十二五  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zh_core_web_sm\n",
    "nlp = zh_core_web_sm.load()\n",
    "corpus = st.CorpusFromPandas(df, category_col='party', text_col='text', nlp=nlp).build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['社会 管理', '加快 转变', '逐步', '转变 经济', '按照', '分开', '积极 发展', '要', '中小企业', '经济 发展', '管理 和', '法规', '农民', '社区', '继续', '军队', '法律 法规', '文化 产业', '发展 方式', '转变', '分配', '事业', '服务业', '群众', '单位', '循环', '科技 创新', '优先', '劳动', '合理', '相 结合', '相', '工资', '其他', '研发', '进一步', '农村', '功能区', '关系', '大力 发展', '变化', '管理 体制', '结合 的', '环境 保护', '的 重要', '科学 发展', '收入 分配', '严格', '税收', '收入', '结合', '要求', '节约', '职能', '中小', '节能', '适应', '原则', '管理', '方针', '的 要求', '循环 经济', '法律', '旅游', '评价', '建立 健全', '国内', '居民', '社会 发展', '结构', '城乡', '社会 组织', '科技', '大力', '经济 社会', '范围', '利用', '评估', '住房', '主体 功能区', '积极', '事业 单位', '灾害', '行政', '培训', '重要', '以及', '保护 和', '高', '规划', '体制 改革', '污染', '和 服务', '生活', '城镇化', '体育', '主要', '两岸', '调整', '节 加强']\n"
     ]
    }
   ],
   "source": [
    "term_freq_df = corpus.get_term_freq_df()\n",
    "term_freq_df['twelve score'] = corpus.get_scaled_f_scores('十二五')\n",
    "Twelve_most = list(term_freq_df.sort_values(by='twelve score',ascending=False).index[:100])\n",
    "print(Twelve_most)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['数据', '贫困', '贫困 地区', '法治', '更 好', '共享', '融合 发展', '精准', '发展 新', '扶贫', '行动', '大 数据', '立法', '绿色', '国家 安全', '融合', '打造', '互联网', '信用', '网络', '全', '智能', '更', '好', '传统', '供给', '权力', '空间', '通道', '资本', '引领', '更 多', '文明', '全球', '维护', '多', '计划', '系统', '深入', '协同', '加快 建设', '好 发挥', '平台', '贸易', '中国', '基金', '必须', '模式', '联动', '开展', '及', '境外', '激励', '自由', '升级', '公开', '生态', '转型', '司法', '高效', '风险', '构建', '一批', '对外', '持续', '参与', '治理 体系', '脱贫', '社会 治理', '治理', '责任', '党', '质量', '全面', '交流', '生态 环境', '设施', '提升', '全面 推进', '大', '中心', '实施', '规则', '安全 保障', '推广', '安全', '基础 设施', '支持', '以上', '全民', '深度', '创新', '人员', '实现', '等 领域', '创业', '支撑', '能', '节 促进', '现代']\n"
     ]
    }
   ],
   "source": [
    "term_freq_df['thirteen score'] = corpus.get_scaled_f_scores('十三五')\n",
    "Thirteen_most = list(term_freq_df.sort_values(by='thirteen score',ascending=False).index[:100])\n",
    "print(Thirteen_most)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1405132"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html = st.produce_scattertext_explorer(corpus, category='十二五', category_name='十二五', not_category_name='十三五',\n",
    "                                      width_in_pixels=1000, metadata=df['speaker'])\n",
    "open(\"scattertext_01.html\",'wb').write(html.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_model = { '文化':['文化','文明'],\n",
    "                '民生':['收入','社区','分配','管理'],\n",
    "                '脱贫':['脱贫','地区','农村','摘帽'],\n",
    "                '生态':['绿色','生态','环境','环保'],\n",
    "                '改革':['发展','转变','深入','循环','创新'],\n",
    "                '科技':['数据','互联网','网络','现代化','科技']\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1059396"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_feature_builder = st.FeatsFromTopicModel(topic_model)\n",
    "\n",
    "topic_corpus = st.CorpusFromParsedDocuments(df,category_col='party',parsed_col='text',feats_from_spacy_doc=topic_feature_builder).build()\n",
    "\n",
    "html = st.produce_scattertext_explorer(\n",
    "    topic_corpus,\n",
    "    category='十二五',\n",
    "    category_name='十二五',\n",
    "    not_category_name='十三五',\n",
    "    width_in_pixels=1000,\n",
    "    metadata=df['speaker'],\n",
    "    use_non_text_features=True,\n",
    "    use_full_doc=True,\n",
    "    pmi_threshold_coefficient=0,\n",
    "    topic_model_term_lists=topic_feature_builder.get_top_model_term_lists()\n",
    ")\n",
    "\n",
    "open('scattertext_02.html','wb').write(html.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from scipy.sparse.linalg import svds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['parse'] = df['text'].apply(st.whitespace_nlp_with_sentences)\n",
    "corpus = (st.CorpusFromParsedDocuments(df,category_col='party',parsed_col='parse').build()\n",
    "         .get_stoplisted_unigram_corpus()\n",
    "         .remove_infrequent_words(minimum_term_count=1,term_ranker=st.OncePerDocFrequencyRanker))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(308, 3) (3,) (3, 36)\n",
      "308\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>term</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>十三五</th>\n",
       "      <td>0.053471</td>\n",
       "      <td>-0.309904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>协调发展</th>\n",
       "      <td>0.035523</td>\n",
       "      <td>-0.201211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>第一章</th>\n",
       "      <td>0.034361</td>\n",
       "      <td>-0.212936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>发展环境</th>\n",
       "      <td>0.034361</td>\n",
       "      <td>-0.212936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>十二五</th>\n",
       "      <td>0.068722</td>\n",
       "      <td>-0.425871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>管办分开</th>\n",
       "      <td>0.011453</td>\n",
       "      <td>0.005024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>强化社会管理和公共服务职能</th>\n",
       "      <td>0.005319</td>\n",
       "      <td>0.008259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>卫生</th>\n",
       "      <td>0.005319</td>\n",
       "      <td>0.008259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>走出去</th>\n",
       "      <td>-0.010540</td>\n",
       "      <td>0.042598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>中医药</th>\n",
       "      <td>-0.033420</td>\n",
       "      <td>0.015779</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>308 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      x         y\n",
       "term                             \n",
       "十三五            0.053471 -0.309904\n",
       "协调发展           0.035523 -0.201211\n",
       "第一章            0.034361 -0.212936\n",
       "发展环境           0.034361 -0.212936\n",
       "十二五            0.068722 -0.425871\n",
       "...                 ...       ...\n",
       "管办分开           0.011453  0.005024\n",
       "强化社会管理和公共服务职能  0.005319  0.008259\n",
       "卫生             0.005319  0.008259\n",
       "走出去           -0.010540  0.042598\n",
       "中医药           -0.033420  0.015779\n",
       "\n",
       "[308 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings = TfidfTransformer().fit_transform(corpus.get_term_doc_mat())\n",
    "corpus.get_num_docs()\n",
    "corpus.get_num_terms()\n",
    "embeddings = embeddings.T\n",
    "\n",
    "U, S, VT = svds(embeddings, k=3, maxiter=20000, which='LM')\n",
    "print(U.shape, S.shape, VT.shape)\n",
    "\n",
    "x_dim=0\n",
    "y_dim=1\n",
    "\n",
    "projection = pd.DataFrame({'term':corpus.get_terms(),'x':U.T[x_dim],'y':U.T[y_dim]}).set_index('term')\n",
    "print(len(projection))\n",
    "projection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/envs/notebook/lib/python3.6/site-packages/scattertext/termscoring/ScaledFScore.py:296: RuntimeWarning: invalid value encountered in true_divide\n",
      "  recall = cat_word_counts * 1. / cat_word_counts.sum()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1151803"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html = st.produce_pca_explorer(corpus,\n",
    "                              category='十二五',\n",
    "                              category_name='十二五',\n",
    "                              not_category_name='十三五',\n",
    "                              projection=projection,\n",
    "                              metadata=df['speaker'],\n",
    "                              width_in_pixels=1000,\n",
    "                              scaler = st.scale_neg_1_to_1_with_zero_mean,\n",
    "                              x_dim=x_dim,\n",
    "                              y_dim=y_dim)\n",
    "open('scattertext_03.html','wb').write(html.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1073036"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#用句子断开文章\n",
    "df['parse'] = df['text'].apply(st.whitespace_nlp_with_sentences)\n",
    "corpus = (st.CorpusFromParsedDocuments(df,category_col='party',parsed_col='parse').build()\n",
    "         .get_stoplisted_unigram_corpus())\n",
    "corpus = corpus.add_doc_names_as_metadata(corpus.get_df()['speaker'])\n",
    "embeddings = TfidfTransformer().fit_transform(corpus.get_term_doc_mat())\n",
    "\n",
    "#SVD降维\n",
    "u, s, vt = svds(embeddings, k=3, maxiter=20000, which='LM')\n",
    "projection = pd.DataFrame({'term':corpus.get_metadata(), 'x':u.T[0], 'y':u.T[1]}).set_index('term')\n",
    "\n",
    "category='十二五'\n",
    "scores = (corpus.get_category_ids() == corpus.get_categories().index(category)).astype(int)\n",
    "\n",
    "html = st.produce_pca_explorer(corpus, \n",
    "                               category=category,\n",
    "                               category_name='十二五',\n",
    "                               not_category_name='十三五',\n",
    "                               metadata=df['speaker'],\n",
    "                               width_in_pixels=1000,\n",
    "                               scaler = st.scale_neg_1_to_1_with_zero_mean,\n",
    "                               show_axes=False,\n",
    "                               use_non_text_features=True,\n",
    "                               use_full_doc=True,\n",
    "                               projection=projection,\n",
    "                               scores=scores,\n",
    "                               show_top_terms=True)\n",
    "open('scattertext_04.html','wb').write(html.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}