LiwenliangSampling.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Data Preprocessing for labeling**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "import plotly.express as px\n",
    "%matplotlib inline"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Loading Dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_35200\\2606980155.py:3: DtypeWarning: Columns (7) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv('dataset/LiwenliangFull.csv')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>max_id</th>\n",
       "      <th>微博id</th>\n",
       "      <th>评论页码</th>\n",
       "      <th>评论id</th>\n",
       "      <th>评论时间</th>\n",
       "      <th>评论点赞数</th>\n",
       "      <th>评论者IP归属地</th>\n",
       "      <th>评论者姓名</th>\n",
       "      <th>评论者id</th>\n",
       "      <th>评论者性别</th>\n",
       "      <th>评论者关注数</th>\n",
       "      <th>评论者粉丝数</th>\n",
       "      <th>评论内容</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>4893821462382076</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4893834767763833</td>\n",
       "      <td>2023-04-23 23:43:14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>来自广西</td>\n",
       "      <td>W我想撑下去</td>\n",
       "      <td>5417297741</td>\n",
       "      <td>男</td>\n",
       "      <td>60</td>\n",
       "      <td>63</td>\n",
       "      <td>朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4893821462382076</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4893834284109853</td>\n",
       "      <td>2023-04-23 23:41:19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>来自湖北</td>\n",
       "      <td>一起看日落吧x</td>\n",
       "      <td>5384707935</td>\n",
       "      <td>女</td>\n",
       "      <td>92</td>\n",
       "      <td>332</td>\n",
       "      <td>李医生 昨天武汉下好大雨哦</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4893821462382076</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4893833496367362</td>\n",
       "      <td>2023-04-23 23:38:11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>来自四川</td>\n",
       "      <td>薄荷甜绿</td>\n",
       "      <td>5255030406</td>\n",
       "      <td>女</td>\n",
       "      <td>169</td>\n",
       "      <td>99</td>\n",
       "      <td>李医生，晚安</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4893821462382076</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4893832905232325</td>\n",
       "      <td>2023-04-23 23:35:50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>来自法国</td>\n",
       "      <td>我是Liquor</td>\n",
       "      <td>2319895202</td>\n",
       "      <td>女</td>\n",
       "      <td>71</td>\n",
       "      <td>21</td>\n",
       "      <td>四月底了 下雨还是那么冷</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4893821462382076</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4893831990086470</td>\n",
       "      <td>2023-04-23 23:32:12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>来自河南</td>\n",
       "      <td>pawyer</td>\n",
       "      <td>6072946222</td>\n",
       "      <td>男</td>\n",
       "      <td>884</td>\n",
       "      <td>242</td>\n",
       "      <td>想起了你</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423466</th>\n",
       "      <td>1423466</td>\n",
       "      <td>4469128381645556</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4469128385866379</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>安水果果菓果果</td>\n",
       "      <td>2251357683</td>\n",
       "      <td>女</td>\n",
       "      <td>477</td>\n",
       "      <td>88</td>\n",
       "      <td>全国人民都在守着你，加油！！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423467</th>\n",
       "      <td>1423467</td>\n",
       "      <td>4469128381645556</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4469128385134746</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>赌神奔奔918</td>\n",
       "      <td>5937000680</td>\n",
       "      <td>女</td>\n",
       "      <td>268</td>\n",
       "      <td>39</td>\n",
       "      <td>加油！我们等你回来🙏🙏🙏</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423468</th>\n",
       "      <td>1423468</td>\n",
       "      <td>4469128381645556</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4469128381647504</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>奔跑的糯米丸子君</td>\n",
       "      <td>6447681726</td>\n",
       "      <td>女</td>\n",
       "      <td>123</td>\n",
       "      <td>30</td>\n",
       "      <td>是假的对不对，能不能出来辟谣一下</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423469</th>\n",
       "      <td>1423469</td>\n",
       "      <td>4469128381645556</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4469128381645557</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>可弟子有惑依旧愚钝</td>\n",
       "      <td>2812656575</td>\n",
       "      <td>女</td>\n",
       "      <td>455</td>\n",
       "      <td>182</td>\n",
       "      <td>求求你，加油加油醒过来！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423470</th>\n",
       "      <td>1423470</td>\n",
       "      <td>4469128381645556</td>\n",
       "      <td>4467107636950632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4469128381651800</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Itachi1u</td>\n",
       "      <td>3101281532</td>\n",
       "      <td>女</td>\n",
       "      <td>112</td>\n",
       "      <td>2179</td>\n",
       "      <td>活下来🙏🏻</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1423471 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unnamed: 0            max_id              微博id  评论页码  \\\n",
       "0                 0  4893821462382076  4467107636950632   1.0   \n",
       "1                 1  4893821462382076  4467107636950632   1.0   \n",
       "2                 2  4893821462382076  4467107636950632   1.0   \n",
       "3                 3  4893821462382076  4467107636950632   1.0   \n",
       "4                 4  4893821462382076  4467107636950632   1.0   \n",
       "...             ...               ...               ...   ...   \n",
       "1423466     1423466  4469128381645556  4467107636950632   NaN   \n",
       "1423467     1423467  4469128381645556  4467107636950632   NaN   \n",
       "1423468     1423468  4469128381645556  4467107636950632   NaN   \n",
       "1423469     1423469  4469128381645556  4467107636950632   NaN   \n",
       "1423470     1423470  4469128381645556  4467107636950632   NaN   \n",
       "\n",
       "                     评论id                 评论时间  评论点赞数 评论者IP归属地      评论者姓名  \\\n",
       "0        4893834767763833  2023-04-23 23:43:14    0.0     来自广西     W我想撑下去   \n",
       "1        4893834284109853  2023-04-23 23:41:19    0.0     来自湖北    一起看日落吧x   \n",
       "2        4893833496367362  2023-04-23 23:38:11    0.0     来自四川       薄荷甜绿   \n",
       "3        4893832905232325  2023-04-23 23:35:50    0.0     来自法国   我是Liquor   \n",
       "4        4893831990086470  2023-04-23 23:32:12    0.0     来自河南     pawyer   \n",
       "...                   ...                  ...    ...      ...        ...   \n",
       "1423466  4469128385866379  2020-02-07 00:31:42    0.0      NaN    安水果果菓果果   \n",
       "1423467  4469128385134746  2020-02-07 00:31:42    0.0      NaN    赌神奔奔918   \n",
       "1423468  4469128381647504  2020-02-07 00:31:41    0.0      NaN   奔跑的糯米丸子君   \n",
       "1423469  4469128381645557  2020-02-07 00:31:41    0.0      NaN  可弟子有惑依旧愚钝   \n",
       "1423470  4469128381651800  2020-02-07 00:31:41    0.0      NaN   Itachi1u   \n",
       "\n",
       "              评论者id 评论者性别  评论者关注数 评论者粉丝数                               评论内容  \n",
       "0        5417297741     男      60     63  朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了  \n",
       "1        5384707935     女      92    332                      李医生 昨天武汉下好大雨哦  \n",
       "2        5255030406     女     169     99                             李医生，晚安  \n",
       "3        2319895202     女      71     21                       四月底了 下雨还是那么冷  \n",
       "4        6072946222     男     884    242                               想起了你  \n",
       "...             ...   ...     ...    ...                                ...  \n",
       "1423466  2251357683     女     477     88                     全国人民都在守着你，加油！！  \n",
       "1423467  5937000680     女     268     39                       加油！我们等你回来🙏🙏🙏  \n",
       "1423468  6447681726     女     123     30                   是假的对不对，能不能出来辟谣一下  \n",
       "1423469  2812656575     女     455    182                       求求你，加油加油醒过来！  \n",
       "1423470  3101281532     女     112   2179                              活下来🙏🏻  \n",
       "\n",
       "[1423471 rows x 14 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('dataset/LiwenliangFull.csv')\n",
    "# Understand the basic information of dataset and the data types involved\n",
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Preproccessing**"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Marked the rows containing EMOJIS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def contains_emoji(text):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.search(text) is not None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_8348\\2671285893.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_count['评论内容'] = df_count['评论内容'].astype(str)\n",
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_8348\\2671285893.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_count['评论时间'] = pd.to_datetime(df_count['评论时间'])\n",
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_8348\\2671285893.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_count['Year_Month'] = df_count['评论时间'].dt.strftime('%Y-%m')\n",
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_8348\\2671285893.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_count['count'] = df_count['评论内容'].apply(lambda x: len(x))\n",
      "C:\\Users\\kidul\\AppData\\Local\\Temp\\ipykernel_8348\\2671285893.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_count['contains_emoji'] = df_count['评论内容'].apply(contains_emoji)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>评论内容</th>\n",
       "      <th>评论时间</th>\n",
       "      <th>Year_Month</th>\n",
       "      <th>count</th>\n",
       "      <th>contains_emoji</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了</td>\n",
       "      <td>2023-04-23 23:43:14</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>33</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>李医生 昨天武汉下好大雨哦</td>\n",
       "      <td>2023-04-23 23:41:19</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>李医生，晚安</td>\n",
       "      <td>2023-04-23 23:38:11</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>四月底了 下雨还是那么冷</td>\n",
       "      <td>2023-04-23 23:35:50</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>想起了你</td>\n",
       "      <td>2023-04-23 23:32:12</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423466</th>\n",
       "      <td>全国人民都在守着你，加油！！</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>14</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423467</th>\n",
       "      <td>加油！我们等你回来🙏🙏🙏</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>12</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423468</th>\n",
       "      <td>是假的对不对，能不能出来辟谣一下</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423469</th>\n",
       "      <td>求求你，加油加油醒过来！</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423470</th>\n",
       "      <td>活下来🙏🏻</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1423471 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      评论内容                评论时间 Year_Month  \\\n",
       "0        朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了 2023-04-23 23:43:14    2023-04   \n",
       "1                            李医生 昨天武汉下好大雨哦 2023-04-23 23:41:19    2023-04   \n",
       "2                                   李医生，晚安 2023-04-23 23:38:11    2023-04   \n",
       "3                             四月底了 下雨还是那么冷 2023-04-23 23:35:50    2023-04   \n",
       "4                                     想起了你 2023-04-23 23:32:12    2023-04   \n",
       "...                                    ...                 ...        ...   \n",
       "1423466                     全国人民都在守着你，加油！！ 2020-02-07 00:31:42    2020-02   \n",
       "1423467                       加油！我们等你回来🙏🙏🙏 2020-02-07 00:31:42    2020-02   \n",
       "1423468                   是假的对不对，能不能出来辟谣一下 2020-02-07 00:31:41    2020-02   \n",
       "1423469                       求求你，加油加油醒过来！ 2020-02-07 00:31:41    2020-02   \n",
       "1423470                              活下来🙏🏻 2020-02-07 00:31:41    2020-02   \n",
       "\n",
       "         count  contains_emoji  \n",
       "0           33           False  \n",
       "1           13           False  \n",
       "2            6           False  \n",
       "3           12           False  \n",
       "4            4           False  \n",
       "...        ...             ...  \n",
       "1423466     14           False  \n",
       "1423467     12            True  \n",
       "1423468     16           False  \n",
       "1423469     12           False  \n",
       "1423470      5            True  \n",
       "\n",
       "[1423471 rows x 5 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_count = df[['评论内容','评论时间']]\n",
    "df_count['评论内容'] = df_count['评论内容'].astype(str)\n",
    "df_count['评论时间'] = pd.to_datetime(df_count['评论时间'])\n",
    "df_count['Year_Month'] = df_count['评论时间'].dt.strftime('%Y-%m')\n",
    "df_count['count'] = df_count['评论内容'].apply(lambda x: len(x))\n",
    "\n",
    "df_count['contains_emoji'] = df_count['评论内容'].apply(contains_emoji)\n",
    "df_count.drop_duplicates(subset='评论内容')\n",
    "df_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False    1338419\n",
      "True       85052\n",
      "Name: contains_emoji, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df_count['contains_emoji'].value_counts())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Split database on containing emojis**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>评论内容</th>\n",
       "      <th>评论时间</th>\n",
       "      <th>Year_Month</th>\n",
       "      <th>count</th>\n",
       "      <th>contains_emoji</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了</td>\n",
       "      <td>2023-04-23 23:43:14</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>33</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>李医生 昨天武汉下好大雨哦</td>\n",
       "      <td>2023-04-23 23:41:19</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>13</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>李医生，晚安</td>\n",
       "      <td>2023-04-23 23:38:11</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>四月底了 下雨还是那么冷</td>\n",
       "      <td>2023-04-23 23:35:50</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>想起了你</td>\n",
       "      <td>2023-04-23 23:32:12</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423466</th>\n",
       "      <td>全国人民都在守着你，加油！！</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>14</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423467</th>\n",
       "      <td>加油！我们等你回来🙏🙏🙏</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>12</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423468</th>\n",
       "      <td>是假的对不对，能不能出来辟谣一下</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423469</th>\n",
       "      <td>求求你，加油加油醒过来！</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423470</th>\n",
       "      <td>活下来🙏🏻</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1423471 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      评论内容                评论时间 Year_Month  \\\n",
       "0        朋友可以借我一顿饭吗。出门在外找工作遇到困难了，肚子好饿。太艰难了 2023-04-23 23:43:14    2023-04   \n",
       "1                            李医生 昨天武汉下好大雨哦 2023-04-23 23:41:19    2023-04   \n",
       "2                                   李医生，晚安 2023-04-23 23:38:11    2023-04   \n",
       "3                             四月底了 下雨还是那么冷 2023-04-23 23:35:50    2023-04   \n",
       "4                                     想起了你 2023-04-23 23:32:12    2023-04   \n",
       "...                                    ...                 ...        ...   \n",
       "1423466                     全国人民都在守着你，加油！！ 2020-02-07 00:31:42    2020-02   \n",
       "1423467                       加油！我们等你回来🙏🙏🙏 2020-02-07 00:31:42    2020-02   \n",
       "1423468                   是假的对不对，能不能出来辟谣一下 2020-02-07 00:31:41    2020-02   \n",
       "1423469                       求求你，加油加油醒过来！ 2020-02-07 00:31:41    2020-02   \n",
       "1423470                              活下来🙏🏻 2020-02-07 00:31:41    2020-02   \n",
       "\n",
       "         count  contains_emoji  \n",
       "0           33           False  \n",
       "1           13           False  \n",
       "2            6           False  \n",
       "3           12           False  \n",
       "4            4           False  \n",
       "...        ...             ...  \n",
       "1423466     14           False  \n",
       "1423467     12            True  \n",
       "1423468     16           False  \n",
       "1423469     12           False  \n",
       "1423470      5            True  \n",
       "\n",
       "[1423471 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>评论内容</th>\n",
       "      <th>评论时间</th>\n",
       "      <th>Year_Month</th>\n",
       "      <th>count</th>\n",
       "      <th>contains_emoji</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>疫情三年我负债25万，在努力还债中🙏</td>\n",
       "      <td>2023-04-23 23:13:59</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>晚安李醫生💐</td>\n",
       "      <td>2023-04-23 22:25:06</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>下雨天🌧️</td>\n",
       "      <td>2023-04-23 18:57:32</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>李医生你好吗希望你在天上开心快乐希望我在这个五一之前所希望的所有事都能如我愿 一定会全部实现...</td>\n",
       "      <td>2023-04-23 15:19:23</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>61</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>文亮医生保佑我全家人还有嘟嘟猫。 嘟嘟宝，在喵星好好生活。爸爸妈妈会再次接你回到人间，回到咱...</td>\n",
       "      <td>2023-04-23 09:42:18</td>\n",
       "      <td>2023-04</td>\n",
       "      <td>53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423453</th>\n",
       "      <td>加油！为你祈祷🙏🙏🙏一定要挺过这关</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423454</th>\n",
       "      <td>加油加油加油加油加油加油好人有好报🙏🙏🙏</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423462</th>\n",
       "      <td>求求您了🙏🙏🙏希望能看到你再发一条微博🙏🙏🙏</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>22</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423467</th>\n",
       "      <td>加油！我们等你回来🙏🙏🙏</td>\n",
       "      <td>2020-02-07 00:31:42</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>12</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423470</th>\n",
       "      <td>活下来🙏🏻</td>\n",
       "      <td>2020-02-07 00:31:41</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>85052 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      评论内容  \\\n",
       "11                                      疫情三年我负债25万，在努力还债中🙏   \n",
       "25                                                  晚安李醫生💐   \n",
       "76                                                   下雨天🌧️   \n",
       "106      李医生你好吗希望你在天上开心快乐希望我在这个五一之前所希望的所有事都能如我愿 一定会全部实现...   \n",
       "143      文亮医生保佑我全家人还有嘟嘟猫。 嘟嘟宝，在喵星好好生活。爸爸妈妈会再次接你回到人间，回到咱...   \n",
       "...                                                    ...   \n",
       "1423453                                  加油！为你祈祷🙏🙏🙏一定要挺过这关   \n",
       "1423454                               加油加油加油加油加油加油好人有好报🙏🙏🙏   \n",
       "1423462                             求求您了🙏🙏🙏希望能看到你再发一条微博🙏🙏🙏   \n",
       "1423467                                       加油！我们等你回来🙏🙏🙏   \n",
       "1423470                                              活下来🙏🏻   \n",
       "\n",
       "                       评论时间 Year_Month  count  contains_emoji  \n",
       "11      2023-04-23 23:13:59    2023-04     18            True  \n",
       "25      2023-04-23 22:25:06    2023-04      6            True  \n",
       "76      2023-04-23 18:57:32    2023-04      5            True  \n",
       "106     2023-04-23 15:19:23    2023-04     61            True  \n",
       "143     2023-04-23 09:42:18    2023-04     53            True  \n",
       "...                     ...        ...    ...             ...  \n",
       "1423453 2020-02-07 00:31:42    2020-02     17            True  \n",
       "1423454 2020-02-07 00:31:42    2020-02     20            True  \n",
       "1423462 2020-02-07 00:31:42    2020-02     22            True  \n",
       "1423467 2020-02-07 00:31:42    2020-02     12            True  \n",
       "1423470 2020-02-07 00:31:41    2020-02      5            True  \n",
       "\n",
       "[85052 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_count_emoji = df_count[df_count['contains_emoji']==True]\n",
    "df_count_no_emoji = df_count[df_count['contains_emoji']==False]\n",
    "display(df_count)\n",
    "display(df_count_emoji)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Sampling and saving**  \n",
    "**Not containing emojis**  \n",
    "First, divide it into layers based on the month.  \n",
    "Then, evenly divide it into five layers from most to least words. Each layer should be randomly sampled at a rate of 0.011%.  \n",
    "In order to maintain the distribution of words.    \n",
    "  **Containing Emojis**  \n",
    "Group by months and then samplling at a rate of 0.011%.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>评论内容</th>\n",
       "      <th>Year_Month</th>\n",
       "      <th>Whailing Wall</th>\n",
       "      <th>Tree Hole</th>\n",
       "      <th>Not related</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>682</td>\n",
       "      <td>嗨 下午好 注定了又是加班的一个下午</td>\n",
       "      <td>2020-09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>967</td>\n",
       "      <td>李医生中午好啊，最近很不开心，打算换工作了但是一时间又不知道自己该不该转变职业方向，很苦恼</td>\n",
       "      <td>2021-06</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>126</td>\n",
       "      <td>起来啊！！快起来！！国家要给你道歉，你要亲耳听见啊！！！</td>\n",
       "      <td>2020-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1543</td>\n",
       "      <td>🙏🙏</td>\n",
       "      <td>2021-02</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1366</td>\n",
       "      <td>保佑您的家人</td>\n",
       "      <td>2022-12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1570</th>\n",
       "      <td>986</td>\n",
       "      <td>怀念李文亮医生</td>\n",
       "      <td>2021-08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1571</th>\n",
       "      <td>1122</td>\n",
       "      <td>想回家，因为疫情不知道能不能顺利回家过年</td>\n",
       "      <td>2022-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1572</th>\n",
       "      <td>399</td>\n",
       "      <td>我好害怕</td>\n",
       "      <td>2020-05</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1573</th>\n",
       "      <td>643</td>\n",
       "      <td>李医生晚安鸭</td>\n",
       "      <td>2020-09</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1574</th>\n",
       "      <td>353</td>\n",
       "      <td>来过，留下脚印。感谢上帝派你来过这个世界！</td>\n",
       "      <td>2020-04</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1575 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                           评论内容 Year_Month  \\\n",
       "0       682                             嗨 下午好 注定了又是加班的一个下午    2020-09   \n",
       "1       967  李医生中午好啊，最近很不开心，打算换工作了但是一时间又不知道自己该不该转变职业方向，很苦恼    2021-06   \n",
       "2       126                   起来啊！！快起来！！国家要给你道歉，你要亲耳听见啊！！！    2020-02   \n",
       "3      1543                                             🙏🙏    2021-02   \n",
       "4      1366                                         保佑您的家人    2022-12   \n",
       "...     ...                                            ...        ...   \n",
       "1570    986                                        怀念李文亮医生    2021-08   \n",
       "1571   1122                           想回家，因为疫情不知道能不能顺利回家过年    2022-01   \n",
       "1572    399                                           我好害怕    2020-05   \n",
       "1573    643                                         李医生晚安鸭    2020-09   \n",
       "1574    353                          来过，留下脚印。感谢上帝派你来过这个世界！    2020-04   \n",
       "\n",
       "      Whailing Wall  Tree Hole  Not related  \n",
       "0                 0          0            0  \n",
       "1                 0          0            0  \n",
       "2                 0          0            0  \n",
       "3                 0          0            0  \n",
       "4                 0          0            0  \n",
       "...             ...        ...          ...  \n",
       "1570              0          0            0  \n",
       "1571              0          0            0  \n",
       "1572              0          0            0  \n",
       "1573              0          0            0  \n",
       "1574              0          0            0  \n",
       "\n",
       "[1575 rows x 6 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_count = df\n",
    "df_count['评论内容'] = df_count['评论内容'].astype(str)\n",
    "df_count['评论时间'] = pd.to_datetime(df_count['评论时间'])\n",
    "df_count['Year_Month'] = df_count['评论时间'].dt.strftime('%Y-%m')\n",
    "df_count['count'] = df_count['评论内容'].apply(lambda x: len(x))\n",
    "\n",
    "df_count['contains_emoji'] = df_count['评论内容'].apply(contains_emoji)\n",
    "df_count.drop_duplicates(subset='评论内容')\n",
    "df_count\n",
    "    \n",
    "grouped = df_count_no_emoji.groupby('Year_Month', as_index=True, group_keys=True)\n",
    "sampled_df = grouped.apply(lambda x:x.sort_values('count',ascending = False))\n",
    "sampled_df['quantile'] = grouped.apply(lambda x: pd.qcut(x['count'], 5, labels=False))\n",
    "sampled_df = sampled_df.apply(lambda x:x.reset_index(drop=True))\n",
    "sampled_df = sampled_df.groupby(['Year_Month', 'quantile'], as_index=True, group_keys=True).apply(lambda x: x.sample(frac = 0.0011))\n",
    "sampled_df = sampled_df.apply(lambda x:x.reset_index(drop=True))\n",
    "sampled_df = sampled_df.drop(['评论时间','count','contains_emoji','quantile'], axis=1)\n",
    "\n",
    "grouped_emoji = df_count_emoji.groupby('Year_Month', as_index=True, group_keys=True)\n",
    "sampled_df_emoji = grouped_emoji.apply(lambda x:x.sample(frac = 0.0011))\n",
    "sampled_df_emoji = sampled_df_emoji.apply(lambda x:x.reset_index(drop=True))\n",
    "sampled_df_emoji = sampled_df_emoji.drop(['评论时间','count','contains_emoji'], axis=1)\n",
    "\n",
    "sample = pd.concat([sampled_df, sampled_df_emoji],axis=0)\n",
    "sample = sample.apply(lambda x:x.reset_index(drop=True))\n",
    "sample = sample.sample(frac=1).reset_index()\n",
    "sample['Whailing Wall'] = 0\n",
    "sample['Tree Hole'] = 0\n",
    "sample['Not related'] = 0\n",
    "sample.to_csv('dataset/LiwenliangSample.csv', index=False, encoding='utf-8-sig')\n",
    "display(sample)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Loop for 4 times and get 4 files, all containing around 1600 rows and label individually.**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kidul",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "8acc9a0bc76a0007f14f7c5879773cd8546098d2fad009824f6fdaab3e99c703"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}