Machine Learning/建製新聞分類器：基礎篇/分類器_基礎篇_hw.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8-final"},"orig_nbformat":2,"kernelspec":{"name":"python_defaultSpec_1597541602087","display_name":"Python 3.6.8 64-bit ('ENV': venv)"},"colab":{"name":"分類器_基礎篇_hw.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"c0066fc6ec594a5a81553dbdc7e2a18e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_e724a3ceabeb4ecc89a114ff25ebce81","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_60e3e995eb4c4fe890f7bdbc1d0aeba2","IPY_MODEL_950f9643993d4700b93669a763baf1cb"]}},"e724a3ceabeb4ecc89a114ff25ebce81":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"60e3e995eb4c4fe890f7bdbc1d0aeba2":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_bb6cb2d2ff6648a4ad309273ede423aa","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":1800,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":1800,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_748537479b9646568f74bb009d2c9a1d"}},"950f9643993d4700b93669a763baf1cb":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_88371987af1147aea9cf95c7736f2c17","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 1800/1800 [02:54&lt;00:00, 10.29it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_a0ff3ec2d35c47cbb5da5df373376ce3"}},"bb6cb2d2ff6648a4ad309273ede423aa":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"748537479b9646568f74bb009d2c9a1d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"88371987af1147aea9cf95c7736f2c17":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"a0ff3ec2d35c47cbb5da5df373376ce3":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"b7bf51c4c8744345a543bc4978091e26":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_13822fa90d7c459c84cb4a0f869a7dcb","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_ccd01dc57a8744ff8317a4045392c89e","IPY_MODEL_2fa3878b17e44508802e49b8e8d80d81"]}},"13822fa90d7c459c84cb4a0f869a7dcb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ccd01dc57a8744ff8317a4045392c89e":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_3bdbc8ff390942b5bf6e0e265bca5685","_dom_classes":[],"description":"100%","_model_name":"FloatProgressModel","bar_style":"success","max":600,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":600,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_72a21ada0b214b8e86b1901b814a48b9"}},"2fa3878b17e44508802e49b8e8d80d81":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_d13729f1445e4d2b9de550c3e7e2c207","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 600/600 [00:55&lt;00:00, 10.85it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_3f30636b9d56443f9b7c3553bbf867da"}},"3bdbc8ff390942b5bf6e0e265bca5685":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"72a21ada0b214b8e86b1901b814a48b9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"d13729f1445e4d2b9de550c3e7e2c207":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"3f30636b9d56443f9b7c3553bbf867da":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"code","metadata":{"tags":[],"id":"txm8AlsqhEdL","executionInfo":{"status":"ok","timestamp":1608121649757,"user_tz":-480,"elapsed":3775,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["!pip install -q -U ckiptagger[tf,gdown]"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"LexwfXqahEdL","executionInfo":{"status":"ok","timestamp":1608121666741,"user_tz":-480,"elapsed":640,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["import numpy as np\n","import pandas as pd\n","\n","from ckiptagger import data_utils\n","from ckiptagger import WS, POS\n","from tqdm.notebook import tqdm"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"R8vd3SG3hEdL","executionInfo":{"status":"ok","timestamp":1608121668101,"user_tz":-480,"elapsed":552,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["df_train = pd.read_csv('news_clustering_train.tsv', sep='\\t')\n","df_test = pd.read_csv('news_clustering_test.tsv', sep='\\t')"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":402},"id":"NtqcARsIhEdM","executionInfo":{"status":"ok","timestamp":1608121669332,"user_tz":-480,"elapsed":678,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"d3ba63af-d576-4e82-ba5a-a6d336924aea"},"source":["df_train"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>index</th>\n","      <th>class</th>\n","      <th>title</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>體育</td>\n","      <td>亞洲杯奪冠賠率：日本、伊朗領銜 中國竟與泰國並列</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>體育</td>\n","      <td>9輪4球本土射手僅次武磊 黃紫昌要搶最強U23頭銜</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>體育</td>\n","      <td>如果今年勇士奪冠，下賽季詹姆斯何去何從？</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>體育</td>\n","      <td>超級替補！科斯塔本賽季替補出場貢獻7次助攻</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>體育</td>\n","      <td>騎士6天里發生了啥？從首輪搶七到次輪3-0猛龍</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1795</th>\n","      <td>1795</td>\n","      <td>遊戲</td>\n","      <td>LOL：麻辣香鍋韓服Rank不合成打野刀？電刀巨魔新套路連勝中</td>\n","    </tr>\n","    <tr>\n","      <th>1796</th>\n","      <td>1796</td>\n","      <td>遊戲</td>\n","      <td>穩住，我們能贏！因為我們擁有這種強大的力量</td>\n","    </tr>\n","    <tr>\n","      <th>1797</th>\n","      <td>1797</td>\n","      <td>遊戲</td>\n","      <td>騰訊是怎樣毀掉《鬥戰神》這款可能成神的作品的？</td>\n","    </tr>\n","    <tr>\n","      <th>1798</th>\n","      <td>1798</td>\n","      <td>遊戲</td>\n","      <td>LOL你不知道的黑科技打法！</td>\n","    </tr>\n","    <tr>\n","      <th>1799</th>\n","      <td>1799</td>\n","      <td>遊戲</td>\n","      <td>遊戲拳皇各代的劇情都在講些什麼？</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1800 rows × 3 columns</p>\n","</div>"],"text/plain":["      index class                            title\n","0         0    體育         亞洲杯奪冠賠率：日本、伊朗領銜 中國竟與泰國並列\n","1         1    體育        9輪4球本土射手僅次武磊 黃紫昌要搶最強U23頭銜\n","2         2    體育             如果今年勇士奪冠，下賽季詹姆斯何去何從？\n","3         3    體育            超級替補！科斯塔本賽季替補出場貢獻7次助攻\n","4         4    體育          騎士6天里發生了啥？從首輪搶七到次輪3-0猛龍\n","...     ...   ...                              ...\n","1795   1795    遊戲  LOL：麻辣香鍋韓服Rank不合成打野刀？電刀巨魔新套路連勝中\n","1796   1796    遊戲            穩住，我們能贏！因為我們擁有這種強大的力量\n","1797   1797    遊戲          騰訊是怎樣毀掉《鬥戰神》這款可能成神的作品的？\n","1798   1798    遊戲                   LOL你不知道的黑科技打法！\n","1799   1799    遊戲                 遊戲拳皇各代的劇情都在講些什麼？\n","\n","[1800 rows x 3 columns]"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"id":"gzp4O8H1hEdM","executionInfo":{"status":"ok","timestamp":1608121674058,"user_tz":-480,"elapsed":1106,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["train_titles = {row['index']: row['title'] for _, row in df_train.iterrows()}\n","train_classes = {row['index']: row['class'] for _, row in df_train.iterrows()}\n","\n","test_titles = {row['index']: row['title'] for _, row in df_test.iterrows()}\n","test_classes = {row['index']: row['class'] for _, row in df_test.iterrows()}"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"7j5SpkvWhEdM","executionInfo":{"status":"ok","timestamp":1608121674818,"user_tz":-480,"elapsed":753,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["all_news_class = ['體育', '財經', '科技', '旅遊', '農業', '遊戲']"],"execution_count":6,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"jxBgy-gLhEdM"},"source":["# 斷詞 + POS"]},{"cell_type":"code","metadata":{"tags":[],"colab":{"base_uri":"https://localhost:8080/"},"id":"a59PmH43hEdN","executionInfo":{"status":"ok","timestamp":1608121831056,"user_tz":-480,"elapsed":141274,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"836aaaa2-8042-43ef-d94f-705bb64b1d30"},"source":["data_utils.download_data_gdown(\"./\")\n","ws = WS('./data/')\n","pos = POS('./data/')"],"execution_count":7,"outputs":[{"output_type":"stream","text":["Downloading...\n","From: https://drive.google.com/uc?id=1efHsY16pxK0lBD2gYCgCTnv1Swstq771\n","To: /content/data.zip\n","1.88GB [00:13, 139MB/s] \n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"tags":[],"colab":{"base_uri":"https://localhost:8080/","height":66,"referenced_widgets":["c0066fc6ec594a5a81553dbdc7e2a18e","e724a3ceabeb4ecc89a114ff25ebce81","60e3e995eb4c4fe890f7bdbc1d0aeba2","950f9643993d4700b93669a763baf1cb","bb6cb2d2ff6648a4ad309273ede423aa","748537479b9646568f74bb009d2c9a1d","88371987af1147aea9cf95c7736f2c17","a0ff3ec2d35c47cbb5da5df373376ce3"]},"id":"-DZOVZ6ahEdN","executionInfo":{"status":"ok","timestamp":1608122007454,"user_tz":-480,"elapsed":176393,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"3417f08c-b9ca-4068-f249-9131fb0d7a49"},"source":["train_title_cuts = {}\n","for index, title in tqdm(train_titles.items()):\n","    word_s = ws([title])\n","    word_p = pos(word_s)\n","    train_title_cuts[index] = list(zip(word_s[0], word_p[0]))"],"execution_count":8,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c0066fc6ec594a5a81553dbdc7e2a18e","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=1800.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"tags":[],"colab":{"base_uri":"https://localhost:8080/","height":66,"referenced_widgets":["b7bf51c4c8744345a543bc4978091e26","13822fa90d7c459c84cb4a0f869a7dcb","ccd01dc57a8744ff8317a4045392c89e","2fa3878b17e44508802e49b8e8d80d81","3bdbc8ff390942b5bf6e0e265bca5685","72a21ada0b214b8e86b1901b814a48b9","d13729f1445e4d2b9de550c3e7e2c207","3f30636b9d56443f9b7c3553bbf867da"]},"id":"_iwyXTDnhEdN","executionInfo":{"status":"ok","timestamp":1608122062622,"user_tz":-480,"elapsed":231553,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"c8e4f44e-5591-4c4e-d999-75225c54d086"},"source":["test_title_cuts = {}\n","for index, title in tqdm(test_titles.items()):\n","    word_s = ws([title])\n","    word_p = pos(word_s)\n","    test_title_cuts[index] = list(zip(word_s[0], word_p[0]))"],"execution_count":9,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"b7bf51c4c8744345a543bc4978091e26","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, max=600.0), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MVvQVjZchEdO","executionInfo":{"status":"ok","timestamp":1608122062622,"user_tz":-480,"elapsed":231547,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"a14d299e-ede7-4f28-efda-65bd1971073b"},"source":["train_title_cuts[120]"],"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('國腳', 'Na'),\n"," ('張呈棟', 'Nb'),\n"," ('：', 'COLONCATEGORY'),\n"," ('從', 'D'),\n"," ('沒', 'D'),\n"," ('想', 'VE'),\n"," ('過', 'Di'),\n"," ('自己', 'Nh'),\n"," ('會', 'D'),\n"," ('出', 'VC'),\n"," ('一', 'Neu'),\n"," ('本', 'Nf'),\n"," ('書', 'Na')]"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"markdown","metadata":{"id":"lSIExKMAhEdO"},"source":["# Bag of Words (BOW)"]},{"cell_type":"code","metadata":{"id":"nfpwJ4KmhEdO","executionInfo":{"status":"ok","timestamp":1608122062623,"user_tz":-480,"elapsed":231540,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["word2index = {}\n","# 產生字與index對應的關係\n","for index in train_title_cuts:\n","    for word, flag in train_title_cuts[index]:\n","        if word in word2index:\n","            continue\n","        word2index[word] = len(word2index)\n","index2word = {val: key for key, val in word2index.items()}"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xY2IGk0ohEdO","executionInfo":{"status":"ok","timestamp":1608122062623,"user_tz":-480,"elapsed":231533,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"efab85a9-2001-48f7-bfa0-e2d3a29c881f"},"source":["word2index['溫暖']"],"execution_count":12,"outputs":[{"output_type":"execute_result","data":{"text/plain":["1520"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"qiBtGcRphEdO","executionInfo":{"status":"ok","timestamp":1608122062623,"user_tz":-480,"elapsed":231526,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"8b44d590-1951-4891-ba56-fdd4544f00c1"},"source":["index2word[1520]"],"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'溫暖'"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"HRRPfba8hEdP","executionInfo":{"status":"ok","timestamp":1608122062624,"user_tz":-480,"elapsed":231525,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["def get_bow_vector(pairs, word2index):\n","    vector = np.zeros(len(word2index))\n","    for word, flag in pairs:\n","        if word in word2index:\n","            vector[word2index[word]] += 1\n","\n","    return vector"],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dbCgKf7whEdP","executionInfo":{"status":"ok","timestamp":1608122062624,"user_tz":-480,"elapsed":230197,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"c5c9e6e8-051c-4f14-e385-d241ab26265b"},"source":["get_bow_vector(train_title_cuts[120], word2index)"],"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([0., 0., 0., ..., 0., 0., 0.])"]},"metadata":{"tags":[]},"execution_count":15}]},{"cell_type":"markdown","metadata":{"id":"vHcp6ghAhEdP"},"source":["# 排除較無意義的詞性"]},{"cell_type":"code","metadata":{"tags":[],"colab":{"base_uri":"https://localhost:8080/"},"id":"R2RTysvphEdP","executionInfo":{"status":"ok","timestamp":1608122063653,"user_tz":-480,"elapsed":1021,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"bc0e723e-9ba5-48b7-a001-eda2a4379c9b"},"source":["pos_analysis = {}\n","for _, pairs in train_title_cuts.items():\n","    for word, flag in pairs:\n","        if flag not in pos_analysis:\n","            pos_analysis[flag] = set()\n","        pos_analysis[flag].add(word)\n","\n","for flag, words in pos_analysis.items():\n","    print(flag, ':', list(words)[:100])\n","    print('=======================')"],"execution_count":16,"outputs":[{"output_type":"stream","text":["Nb : ['電一', '穆謝奎', '史森明', '埃梅里', '科勒·卡戴珊', '馬競', '2018世界杯', '拜仁續', '朱之文', '拉基蒂奇', '樂福', '凱爾特', '章澤天', '里坦克', '娜迦王海伯倫', '扎哈維', '張良咸', '克羅托內 ', '武磊', '尼爾 ', '斯威', '密子君', '劉國梁', '勒布朗', '纏中', '秦緒文', '馬龍', '王者榮', '小強', '微星', '安德森', 'm15', '泰拳王西', '科斯塔', '男乒', '索尼', '猛', '·龜茲', '斐濟', '費基爾', '單薇恩', '加威廉', '尤納斯', '四阿哥', '魅族', '微醫', '寶二', '李嘉誠', '雲聯惠', '哈登', '奧拉朱旺遠', '北京人和', '傑尼亞', '皇馬', '德羅贊', '華峰', '扎堆', '阿木木歷', '璞麗', '伊亞', '奧拉朱旺', '安永', '馬內', '王者榮耀', '溫格', '建成', '富力', '何享健', '林美景', '萬孚', '朱嘯虎', '朱婷', '恆大', '雷霆隊', '瓜迪奧拉', '切爾西', '小詹皇', '阿里巴巴', '伏爾加格勒', '世界杯', '億航', '騰訊王', '興業', '安徒恩比盧克', '微眾', '后羿', '樊振東', '巴菲特', '濃眉哥', '高曉松', '詹皇', '星巴克', '查理芒格', '金英權', '周雲傑', '石川佳純', '刷安徒恩', '米切爾西蒙斯', '英超', '神武']\n","=======================\n","VC : ['替補', '放大', '看直', '存', '管理', '打', '打爆', '區分', '舉辦', '毀滅', '擺', '解決', '開復', '買賣', '製造', '購買', '評價', '生產', '報', '抽', '定', '駕馭', '認養', '構建', '纏', '營銷', '承包', '贊美', '通過', '設置', '拍到', '登頂', '打通', '養恩', '結', '發', '錄制', '取捨', '伴隨', '賞', '陪', '攀爬', '罵', '宣一', '解', '取消', '閃', '盤點', '栽種', '點', '打磨', '打響', '面試', '參透', '戲耍', '簽', '開放', '高產', '餵', '引進', '擺出', '購', '反包', '等', '踢出', '觀賞', '收取', '縮減', '守護', '認可', '下載', '調整', '改種', '食用', '打掃', '影響', '完成', '破', '進化到', '建立', '寫', '動用', '投', '燒', '攝', '參考', '隱藏', '渲染', '規劃', '挖', '擺脫', '吊打', '籌劃', '發起', '弄崩', '接', '冠絕', '套養', '練到', '大賺']\n","=======================\n","Na : ['牛股', '盆景', '元素', '選手', '人類', '新品', '爆文', '老人', '潮服', '身價', '現實', '資訊', '裝備', '高地', '牧歌', '烏龜', '飲水', '霸', '真因', '驍龍', '營銷', '紅心', '性價', '使用率', '紅豆', '閃電狼', '大仙', '俗話', '閃電', '大米', '精品', '樹莓', '油價', '竹林', '土地', '開心果', '玄鳳', '秘訣', '證券', '小額', '手藝', '合規性', '糧食', '大叔', '情況', '玻璃', '高峰', '塔', '宜興人', '夢淚', '運動', '水產', '惡魔', '能力', '環保', '關鍵', '漫畫', '基本', '運動員', '花式', '生意', '墊腳', '鐵路篇', '蜂', '遊戲本', '氣味', '綠色', '陣容', '網商貸', '英雄級', '德國隊', '場景', '紅綠燈', '農宿', '後衛', '成功率', '美味', '火線', '要聞', '孩子', '腳', '薪種', '油菜花', '羊', '盤', '收益率', '房產證', '蠱', '小伙', '飼料', '午餐', '菜', '行動', '刀塔', '額度', '母豬', '紀錄', '本金', '性能', '網']\n","=======================\n","COLONCATEGORY : [':', '：']\n","=======================\n","Nc : ['深圳', '四國', '山西', '太湖', '香港', '盧氏縣', '臨洮', '意大利', '冰城', '黃鶴樓', '德', '評論區', '哈尼梯田', '廣元', '實驗室', '長白山', '南寧', '風景道', '火山口', '耿店村', '沿途', '西安', '廣州', '聯合村', '上川島', '洗車店', '天際', '泰安', '賓館', '墾丁', '農科院', '天下', '中棉', '債市', '泰國', '五一', '盱眙', '農業園', '綿竹', '利希', '火焰山', '九寨', '海南', '嵐山', '烏魯木齊', '邊緣', '西藏', '萊加內斯', '母公司', '麥當勞', '央視', '浦口', '目的地', '吉隆坡', '內內', '延邊', '萊茵河', '球館', '咖啡蘭', '華中區', '嘉陵江', '唐人街', '風景區', '西安比', '市場', '國外', '聖堂山', '原地', '湖北', '南沙河鎮', '馬刺', '江夏鎮', '加州', '尼泊爾', '波密', '海口', '切爾西', '柯橋', '金融城', '國', '軍艦島', '馬德里', '牛市', '地上', '工廠', '渾江區', '分行', '瑞士', '便利店', '京東', '武漢', '貴州', '河南', '海岸', '倫貝爾', '超京', '支行', '鵬城', '國內', '世界']\n","=======================\n","PAUSECATEGORY : ['、']\n","=======================\n","VB : ['入賬', '退改', '融資', '充氣', '充飢', '罵屌絲', '收回來', '上架', '領銜', '退款', '加起來', '退貨', '定位', '加油', '分組', '解惑', '計算出來', '分段', '護航', '提前', '騙錢', '鋪平', '一網打盡', '加持', '執教', '加冕', '通話', '應對', '解圍', '代打', '開戰', '再見', '相比', '助威', '站台', '復仇', '解體', '解密', '扣分', '拍照', '加工', '導航', '抓來', '罰款', '鍍金', '分紅', '效力']\n","=======================\n","WHITESPACE : [' ']\n","=======================\n","D : ['或許', '仍然', '剛剛', '需', '越', '多', '不可以', '永', '盡', '大都', '結伴', '均', '不用', '能夠', '能', '永遠', '重新', '別', '陸續', '沒想到', '是否', '不能', '始', '預先', '沒有', '好在', '為何', '無可', '一直', '突然', '同', '專職', '可', '及時', '從未', '絕對', '直', '正', '每', '總算', '一頭', '多多', '即', '原來', '簡直', '只得', '屢', '總是', '只是', '從中', '怎麼', '只有', '全面', '為啥', '全力', '本', '該不該', '接著', '可以', '迫不及待', '居然', '親自', '一定', '真是', '並', '竟然', '很少', '先', '就', '現', '不得', '不', '不再', '冒死', '曾經', '不可', '不會', '無法', '這麼', '相', '親身', '超級', '獨', '卻', '甚至', '常常', '從', '至今', '去', '一手', '略', '未', '已然', '全程', '究竟', '即將', '盡快', '剛', '快', '天南地北']\n","=======================\n","P : ['自從', '憑', '於', '對', '到', '離', '距離', '給', '用', '靠', '向', '至', '就', '比如', '繼', '如', '在', '將', '根據', '往', '和', '隨了', '為了', '等', '被', '從', '比', '除了', '跟', '幫', '以', '直至', '隨', '為', '關於', '慘遭', '受', '除', '當', '針對', '趁著', '像', '自', '按', '因', '把', '待', '趁', '與', '臨', '遭']\n","=======================\n","VG : ['叫', '變成', '合成', '作為', '堪稱', '封號', '譽為', '排行', '彷彿', '升任', '做', '當作', '集成', '建成', '稱', '成為', '稱為', '化', '屈居', '理解為', '如', '變', '充當', '形成', '酷似', '戰成', '居', '算', '誤認成', '投票成', '疑似', '竟是', '評選', '類似', '似', '猶如', '做成', '改造成', '姓', '為', '賣成', '美成', '稱得上', '排', '當', '稱作', '算是', '等於', '名叫', '像', '當成', '並列', '非', '排名', '叫做', '成', '應聘', '屬於', '當上']\n","=======================\n","Neu : ['730萬', '10010', '萬', '！2018', '數千萬', '54', '100', '9-1', '2018千', '72', '1299', '2018-05-06 ', '7億', '1300', '7', '1.8億', '700', '835', '38', '98', '1.3萬', '1.1.2', '第七', '47億', '14億', '8', '42', '一千', '3.5', '2200', '1099', '幾千', '392億', '76', '一百', '100萬', '21', '5050', '十幾', '千', '十三', '4.5億', '3100', '七八千', '500', '第1', '5', '2000萬', '十萬', '56', '2萬', '二', '115', '三百', '17億', '9', '6', '六', '300萬', '3000', '七', '23萬一', '第9', '1688', '48', '50多', '1699', '1268', '第114', '2', '14', '29', '5.6千三', '200', '億', '2018', '千千萬', '30', '0.2', '30萬', '四五', '12', '幾', '90', '（四）', '11.59億', '2:3', '9-0', 'b120', '兩萬', '1.46億', ' 360', '150', '46萬', '1900億', '五一', '17', '九', '16', '24']\n","=======================\n","Nf : ['台', '英吋', '頁', '掛', '輪', '美元', '票', '分鐘', '處', '股', '名', '款', 'K', '本', '口', '槍', '頭', '晚', '點', '番', '則', '招', '步', '戶', '碗', '頓', '樣', '公里', '關', '鍋', '部', '個', '周', '方', '輛', '道', '畝', '屆', '年', '把', '隻', '項', '隊', '港元', '噸', '歲', '斤', '筆', '遍', '群', '串', '張', '度', '色', '班', '米', '棵', '件', '秒', '盤', '期', '回', '座', '袋', '株', '次', '家', '號', '所', '類', '場', '日', '片', '局', '毛', '對', '分', '元', '行', '檔', '條', '週', '級', '支', '門', '句', '塊', '節', '只', '套', '刻', '胎', '位', '層', '種', '路', '篇', '天', '些', '批']\n","=======================\n","VJ : ['看不過去', '惹不起', '抗', '不如', '含有', '錯失', '吃上', '連接', '贏下', '負於', '心疼', '領跑', '沒有', '強過', '耗', '領略', '爆發', '緊缺', '失', '降至', '受到', '領先', '沈迷', '為', '僅次於', '受', '服', '發生', '認', '歸屬於', '順應', '發揮', '熱心', '經歷', '偏愛', '減持', '獨佔', '嫌棄', '脫', '欣賞', '來自', '不到', '惜', '懷念', '缺', '喪失', '冷落', '過完', '恪守', '佔', '減', '平', '不理', '出於', '珍惜', '無', '出', '毫無', '錯過', '自有', '蘊藏', '享受', '符合', '獲', '沒', '負', '恭喜', '認識', '入圍', '得', '大勝', '持有', '替代', '高達', '翻倍', '展望', '處於', '失去', '靠', '超過', '達', '贊同', '出身', '忘懷', '逾', '適用', '達到', '忽略', '賽', '擁有', '年入', '看好', '不敵', '身臨', '歡迎', '優於', '曬', '同情', '缺少', '煥發']\n","=======================\n","VD : ['售', '還', '發給', '禁售', '給予', '分', '送', '寄予', '給', '借', '坑', '補貼', '速遞', '發放', '郵給', '輸', '貸款', '贏', '付', '清繳', '配', '輸給', '交付', '支付', '轉讓', '賣', '欠', '發', '供應', '借貸', '拒賠', '白送', '傳', '提供', '補繳', '予', '貸', '搶', '收費']\n","=======================\n","Dfa : ['多', '還', '不大', '遠', '如此', '有些', '越來越', '好', '比較', '很', '這麼', '同樣', '最', '超', '非常', '更為', '遠遠', '過於', '十分', '更', '較', '頗', '有點', '過', '還要', '太', '極']\n","=======================\n","VH : ['遲滯', '多', '激增', '狂野', '窘', '頭疼', '免費', '出爐', '立體', '無緣', '不錯', '色綠', '復活', '出局', '芳香', '忙', '發愁', '年久失修', '唯美', '傻眼', '養老', '便宜', '眾多', '高效', '見效', '跌停', '奇妙', '上了年紀', '肺腑獨白', '怎麼辦', '心酸', '史無前例', '變差', '積雪', '慢', '得寸進尺', '真實', '通車', '開花', '重重', '安全', '智障', '涼', '原始', '飽', '很少', '超出口', '猛', '過人', '秀麗', '完勝', '現', '那麼多', '驚恐', '空洞', '靠譜', '減產', '爆爆', '霸氣', '簡約', '火爆', '中興', '偉大', '豐收', '成年', '咸', '苦不堪言', '流暢', '爆紅', '中毒', '美味', '新', '優秀', '勝利', '未果', '長', '渾濁', '發達', '聰明', '相反', '直接', '稱霸絕', '幸福', '神奇', '迷路', '濕潤', '獲利', '完善', '濃郁', '還可以', '常見', '賞心悅目', '免息', '知名', '客平人和', '長滿', '多彩', '夠多', '低調', '健康']\n","=======================\n","Cbb : ['否則', '既', '跟著', '雖然', '但', '並', '不如', '縱使', '反而', '連', '即使', '可是', '還不如', '若', '既然', '要是', '就算', '而是', '但是', '假如', '不管', '因為', '不僅', '不然', '並且', '如果', '而', '因', '只要', '不過', '只有', '另外']\n","=======================\n","Nd : ['四月', '過去', '5月', '90年', '現代', '1日', '月季', '-5', '中旬', '週一', '現在', '長期', '00年', '31號', '短期', '15日', '旅遊節', '五月', '4日', '後期', '六', '晚上', '週二', '花季', '相親節', '13點', '採摘節', '今日', '2018', '目前', '五月份', '夏天', '夜', '2019年', '8日', '明年', '同時', '九月', '幾月份', '立夏', '首日', '春季', '2018年', '正月', '最近', '12日', '周', '中期', '小時候', '多久', '昨天', '2017年', '3月', '文化節', '6日', '一大早', '15年', '10點', '七月份', '三國', '秋', '今年', '2005年', '當前', '5.7日', '半夜', '週末', '最後', '寒武紀', '荒古', '夏季', '2035年', '隔年', '昔日', '今夏', '5日', '那時', '06日', '後來', '未來', '一會', '大中', '初夏', '晚清', '夏窗', '晚間', '當時', '狗年', '明晚', '春', '6旬', '19年', '2019', '以前', '近日', '後', '4月', '5號', '7日', '明日']\n","=======================\n","VA : ['隕落', '下馬', '會師', '問津', '看過來', '封卡', '變通', '做生意', '壓哨', '掙扎', '養家糊口', '登高', '血刃', '日落', '簽約', '開門紅', '保級', '扣賬', '旅行', '轉農', '建功', '上場', '摘帽', '養花', '回落', '撒沙', '格鬥', '轉賬', '起航', '作證', '墊腳', '還款', '牧育', '倒伏', '睡', '遠行', '爆錘', '伸手', '報名', '立刷', '旅遊', '釣魚', '領頭', '潛伏', '打團', '大小便', '下卡', '觀戰', '生活', '徜徉', '行走', '開業', '開課', '點贊', '進去', '點球', '面對面', '入手', '跳舞', '追債', '減稅', '做種', '打工', '喜極而泣', '育秧', '實習', '生財', '輪轉', '下海', '出門', '打卡', '喧囂', '結果', '關機', '嘗鮮', '換股', '看盤', '行', '違法', '搭檔', '加菜', '閉嘴', '下班', '字字珠璣', '跑', '騎車', '游水', '何去何從', '革命', '艷遇', '播種', '卡', '出來', '反彈', '答復', '拼房', '退票', '露營', '掉幀', '抄底']\n","=======================\n","COMMACATEGORY : [',', '，']\n","=======================\n","Nes : ['首', '本', '何', '上', '另', '某', '下', '前', '頭', '後', '該', '各', '同', '貴', '每', '近']\n","=======================\n","QUESTIONCATEGORY : ['？']\n","=======================\n","EXCLAMATIONCATEGORY : ['!', '！']\n","=======================\n","Di : ['過', '著', '了', '起來', '個']\n","=======================\n","Nep : ['那', '什麼', '啥', '哪', '其', '其中', '這', '此']\n","=======================\n","Caa : ['比', '又', '跟', 'VS', '到', '或', '暨', '及', '和', '還是', '與', '至', '或是', '或者']\n","=======================\n","VCL : ['入', '留在', '前往', '重返', '抵達', '進入', '到', '來到', '重回', '存進', '出發到', '跌出', '跌至', '穿透', '住', '栽進', '蒞臨', '到達', '走進', '走到', '在', '升至', '走訪', '遠離', '往', '回', '橫掃', '去', '回歸', '用於', '走過', '出', '暗渡', '深入', '遊覽', '穿越', '退出', '登陸', '赴', '上', '進', '穿過', '路過', '直達', '過', '前來', '回到', '盤踞', '走下', '度過', '玩遍']\n","=======================\n","Neqa : ['多', '9.89%', '67%', '更多', '遍', '有些', '任何', '步步', '41.8%', '這麼多', '哪些', '一處處', '無數', '80%', '部分', '所有', '其他', '15%', '其它', '一點', '百分之八十', '四成', '1%', '8%', '一切', '許多', '大量', '個個', '大多', '三分之一', '多點', '半', '90%', '這些', '30%', '100%', '不少', '那些', '全', '一半', '句句', '有的', '1/4', '大多數', '滴滴', '10%', '0.8%', '多少', '一些', '很多', '12%', '6%', '20%']\n","=======================\n","Nh : ['誰', '父子倆', '別人', '自身', '屌絲', '你們', '自我', '她們', '他們', '俺們', '各自', '對方', '一方', '他', '您', '私人', '你', '人家', '我', '大家', '個人', '它們', '我們', '小眾', '她', '本人', '各位', '它', '雙方', '自己']\n","=======================\n","FW : ['Rookie', 'BAT', '6 10.5', 'vs', 'Alan', '6G+', 'Q', 'T+0', 'PPP', 'dota', 'raid', 'Cotswolds', ' 樂', 'i7', 'Kitty', 'DW', 'King James', 'Rank', 'QG', 'mix2', 'ADC', ' + ', 'NBA ', 'DNF：', 'app', 'GPP', 'XQ', ' CCR', '5G', 'mAh+', 'Xbox ', 'SSG', 'Uzi', 'mc', 'adc', 'CCTV5', ' VS ', 'BCH', 'IP', 'vip', 'NPC', 'Lol里', 'CCR', 'RPG', 'Disneyland', 'h', 'CP', ' 展', 'K', 'AP', 'KV', 'X', '+7', 'EOS', '：AG', 'VR', '9A91', 'CDR', 'Kakao', 'qq', 'Instagram', '2', 'VS', 'SIo', 'RNG', 'Minecraft', 'B', ' vs 利物浦', 'WF', '！DOTA2 MOD', 'LOL：', 'wifi', 'DNF', '-0', 'kpl', 'FF ', 'S11', 'MARVEL X', 'G+', 'e', 'Hello ', 'QM', 'Ghappy', '-6', 'MSI', 'Oeanc', 'APP', '+4', 'iphone6plus', ' 利', 'AWM', 'TPP', 'iBoy', 'PS4/NS', 'steam', 'ST', 'Credo', 'VR ', 'TT', 'LPL']\n","=======================\n","SHI : ['是', '像是']\n","=======================\n","VE : ['盛贊', '訴說', '披露', '看看', '公認', '抱怨', '認為', '提猜', '示範', '看出', '反對', '提到', '檢測', '預期', '廣播', '查找', '回答', '證明', '申卡', '發誓', '研究', '估值', '形容', '以為', '爆料', '傳聞', '判定', '想', '討說', '精准', '賭約', '想好', '看到', '告訴', '想起', '禁止', '預計', '調研', '談', '公示', '想到', '宣佈', '證', '深思', '假設', '鑒定', '慶祝', '猜', '判罰', '查看', '暗示', '預測', '介紹', '聯想', '發現', '想要', '做到', '稱', '探索', '聽說', '防止', '公告', '講解', '承諾', '回顧', '聽到', '問', '呼', '考慮', '幻想', '感嘆', '要', '准', '探討', '提起', '查', '判斷', '安排', '提', '提醒', '商量', '講', '聽', '自嘲', '知詢', '限制', '報道', '預定', '見', '見證', '算', '設想', '避免', '炫耀', '呼籲', '確認', '警惕', '看清楚', '公佈', '說明']\n","=======================\n","DE : ['之', '得', '的', '地']\n","=======================\n","VHC : ['溫暖', '增長', '國際化', '增 ', '結束', '可憐', '機構化', '現代化', '恢復', '暫停', '告破', '豐富', '淡化', '普及', '壞', '智能化', '產生', '降低', '停', '優化', '穩定', '還原', '縮小', '感動', '斷', '下來', '煩', '傷', '肥', '加快', '減少', '愁死', '震驚', '麻煩', '強化', '增強', '破', '陷', '增加', '滿足', '開', '冷卻', '繁榮', '凍', '平衡', '擴張', '辛苦', '愁壞', '公開', '團結', '變動', '規模化', '熱']\n","=======================\n","Ng : ['內', '般', '以下', '初', '之內', '底', '前', '時', '間', '之中', '之上', '起', '以上', '之後', '之前', '以外', '以來', '中', '為止', '裡', '外', '上', '之外', '之下', '來', '下', '後', '而言', '之間', '左右', '以後']\n","=======================\n","VK : ['注意', '意外', '需', '欲', '願意', '懂得', '奇怪', '懷疑', '想見', '感謝', '討厭', '曲解', '瞭解', '回憶', '所謂', '需要', '擔心', '不忘', '依賴', '意味', '信', '堅持', '喜歡', '感覺', '遺憾', '顯示', '期待', '知', '希望', '肯定', '害怕', '不滿', '理解', '介意', '遇', '意想不到', '小心', '恐', '放心', '恨', '知道', '忘記', '忘', '明白', '感覺到', '感受', '抱歉', '願', '懂', '確定', '覺得', '怕', '同意', '覺', '盼', '有利於', '流傳', '指望', '面臨']\n","=======================\n","Ncd : ['兩旁', '西部', '對面', '內', '哪兒', '畔', '哪裡', '以下', '底部', '東部', '中間', '當中', '一帶', '這裡', '外面', '邊', '間', '前沿', '西', '上面', '上方', '旁邊', '哪', '方', '中', '裡', '東', '外', '東方', '哪下', '上', '裡面', '下', '中部', '北', '那邊', '末端', '那裡', '東西部']\n","=======================\n","A : ['眾', '優質', '業餘', '唯一', '單', '額外', '現行', '歐式', '新興', '同等', '共同', '後置', '主', '多餘', '上上', '野生', '高度', '新一代', '現役', '獨家', '常務', '基礎', '乒乓', '長途', '超級', '大幅度', '國產', '超', '火紅', '最佳', '總', '頭號', '同一', '彩色', '二手', '重磅', '基層', '碧藍', '一般', '真正', '獨門', '中', '前置', '必備', '寶有', '原', '小本', '獨角', '雙', '非', '麻辣', '綜合', '地下', '中西']\n","=======================\n","V_2 : ['有']\n","=======================\n","VL : ['開始', '能', '甘願', '後悔', '使得', '愛', '愛好', '便利', '使', '趨於', '讓', '持續', '喜好', '致', '敢', '用來', '肯', '不惜', '令', '習慣', '導致', '故意']\n","=======================\n","T : ['去', '吧', '來', '哦', '啊', '啦', '嗎', '麼', '哪', '的', '呢', '矣', '了', '嘛', '咯']\n","=======================\n","VI : ['取勝', '專用', '傾心', '著稱', '驚艷']\n","=======================\n","DASHCATEGORY : ['-', '—']\n","=======================\n","I : ['瑪法里奧', '哦']\n","=======================\n","Dk : ['那', '那麼', '反過來', '據聞', '外傳', '結果', '據說', '想不到']\n","=======================\n","VF : ['叫', '要求', '止住', '帶領', '試試', '擬', '推選', '圖', '拒', '助', '邀', '停止', '告誡', '催', '打算', '拒絕', '試', '率領', '申請', '勸說', '逼', '判', '繼續', '鼓勵', '準備', '策劃', '勸住', '請']\n","=======================\n","Da : ['光', '最多', '幾近', '約', '將近', '幾乎', '只', '足足', '近', '唯獨', '共', '正好', '才', '僅', '不下']\n","=======================\n","PARENTHESISCATEGORY : ['（', '」', '(', '》', '《', '）', '「']\n","=======================\n","Nv : ['銷售', '組合', '私募', '管理', '抗戰', '推薦', '航海', '分銷', '射擊', '登山', '解救', '度假', '配置', '護髮', '思考', '生產', '買單', '問', '增發', '營銷', '絕殺', '回歸', '開發', '創業', '關聯', '供電', '惡搞', '輔助', '訴訟', '體驗', '兼職', '翻譯', '防治', '錄制', '代理', '遊樂', '燃燒', '研究', '判罰', '疊加', '旅遊', '工作', '經歷', '崩潰', '批發', '灌籃']\n","=======================\n","PERIODCATEGORY : ['。', '·']\n","=======================\n","VAC : ['蒸發', '推進', '上演', '集合', '移動', '動']\n","=======================\n","Dfb : ['極了', '一點', '些']\n","=======================\n","Cab : ['等等', '等']\n","=======================\n","SEMICOLONCATEGORY : ['；']\n","=======================\n","Neqb : ['幾', '以上', '多', '半']\n","=======================\n","ETCCATEGORY : ['…']\n","=======================\n","Cba : ['的話']\n","=======================\n","DOTCATEGORY : ['.']\n","=======================\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"gZNKKUCEhEdP"},"source":["|         Type        |     Description    |\n","|:-------------------:|:------------------:|\n","| A                   | 非謂形容詞         |\n","| Caa                 | 對等連接詞         |\n","| Cab                 | 連接詞，如：等等   |\n","| Cba                 | 連接詞，如：的話   |\n","| Cbb                 | 關聯連接詞         |\n","| D                   | 副詞               |\n","| Da                  | 數量副詞           |\n","| Dfa                 | 動詞前程度副詞     |\n","| Dfb                 | 動詞後程度副詞     |\n","| Di                  | 時態標記           |\n","| Dk                  | 句副詞             |\n","| DM                  | 定量式             |\n","| I                   | 感嘆詞             |\n","| Na                  | 普通名詞           |\n","| Nb                  | 專有名詞           |\n","| Nc                  | 地方詞             |\n","| Ncd                 | 位置詞             |\n","| Nd                  | 時間詞             |\n","| Nep                 | 指代定詞           |\n","| Neqa                | 數量定詞           |\n","| Neqb                | 後置數量定詞       |\n","| Nes                 | 特指定詞           |\n","| Neu                 | 數詞定詞           |\n","| Nf                  | 量詞               |\n","| Ng                  | 後置詞             |\n","| Nh                  | 代名詞             |\n","| Nv                  | 名物化動詞         |\n","| P                   | 介詞               |\n","| T                   | 語助詞             |\n","| VA                  | 動作不及物動詞     |\n","| VAC                 | 動作使動動詞       |\n","| VB                  | 動作類及物動詞     |\n","| VC                  | 動作及物動詞       |\n","| VCL                 | 動作接地方賓語動詞 |\n","| VD                  | 雙賓動詞           |\n","| VF                  | 動作謂賓動詞       |\n","| VE                  | 動作句賓動詞       |\n","| VG                  | 分類動詞           |\n","| VH                  | 狀態不及物動詞     |\n","| VHC                 | 狀態使動動詞       |\n","| VI                  | 狀態類及物動詞     |\n","| VJ                  | 狀態及物動詞       |\n","| VK                  | 狀態句賓動詞       |\n","| VL                  | 狀態謂賓動詞       |\n","| V_2                 | 有                 |\n","|                     |                    |\n","| DE                  | 的之得地           |\n","| SHI                 | 是                 |\n","| FW                  | 外文               |\n","|                     |                    |\n","| COLONCATEGORY       | 冒號               |\n","| COMMACATEGORY       | 逗號               |\n","| DASHCATEGORY        | 破折號             |\n","| DOTCATEGORY         | 點號               |\n","| ETCCATEGORY         | 刪節號             |\n","| EXCLAMATIONCATEGORY | 驚嘆號             |\n","| PARENTHESISCATEGORY | 括號               |\n","| PAUSECATEGORY       | 頓號               |\n","| PERIODCATEGORY      | 句號               |\n","| QUESTIONCATEGORY    | 問號               |\n","| SEMICOLONCATEGORY   | 分號               |\n","| SPCHANGECATEGORY    | 雙直線             |\n","| WHITESPACE          | 空白               |"]},{"cell_type":"code","metadata":{"id":"kX7CV3pPhEdQ","executionInfo":{"status":"ok","timestamp":1608122063654,"user_tz":-480,"elapsed":1019,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["def get_bow_vector_with_selection(pairs, word2index):\n","    excluded_flags = [\n","        # 根據以上列舉出來的文字以及詞性表，請列出想要排除的詞性\n","        'Nh', 'Nep', 'Nes', 'DE', 'T', 'P', 'V_2', 'SHI',\n","        'Dfa', 'Dfb', 'Da', 'Di', 'Dk',\n","        'Caa', 'Cab', 'Cba', 'Cbb',\n","        'COLONCATEGORY', 'COMMACATEGORY', 'DASHCATEGORY', 'DOTCATEGORY', 'ETCCATEGORY', 'EXCLAMATIONCATEGORY',\n","        'PARENTHESISCATEGORY', 'PAUSECATEGORY', 'PERIODCATEGORY', 'QUESTIONCATEGORY', 'SEMICOLONCATEGORY',\n","        'SPCHANGECATEGORY', 'WHITESPACE'\n","    ]\n","    vector = np.zeros(len(word2index))\n","    for word, flag in pairs:\n","        if word in word2index and flag not in excluded_flags:\n","            vector[word2index[word]] += 1\n","\n","    return vector"],"execution_count":17,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"yBDszruMhEdQ"},"source":["# Cosine Similarity"]},{"cell_type":"code","metadata":{"id":"x7C9AysMhEdQ","executionInfo":{"status":"ok","timestamp":1608122063654,"user_tz":-480,"elapsed":1010,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["cosine_similarity = lambda x, y: np.dot(x, y) / (np.linalg.norm(x) * np.linalg.norm(y))"],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_hUyrtFwhEdQ","executionInfo":{"status":"ok","timestamp":1608122063655,"user_tz":-480,"elapsed":1004,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"f822d8f8-3cc4-4bca-e09f-da3153b22255"},"source":["bow1 = get_bow_vector(train_title_cuts[100], word2index)\n","bow2 = get_bow_vector(train_title_cuts[130], word2index)\n","cosine_similarity(bow1, bow2)"],"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.08703882797784893"]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"c_mdpU43hEdQ","executionInfo":{"status":"ok","timestamp":1608122064096,"user_tz":-480,"elapsed":1438,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"c2e009e5-438a-40ff-d0e8-21bae63b1e53"},"source":["train_title_cuts[100]"],"execution_count":20,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('山東', 'Nc'),\n"," ('魯能', 'Nb'),\n"," ('有沒有', 'D'),\n"," ('可能', 'D'),\n"," ('拿到', 'VC'),\n"," ('今年', 'Nd'),\n"," ('的', 'DE'),\n"," ('中', 'A'),\n"," ('超', 'A'),\n"," ('冠軍', 'Na'),\n"," ('？', 'QUESTIONCATEGORY')]"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rwFOMPAohEdR","executionInfo":{"status":"ok","timestamp":1608122064097,"user_tz":-480,"elapsed":1433,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"20bfcbd7-b62f-4df6-ad01-34b69d7fe6e3"},"source":["train_title_cuts[130]"],"execution_count":21,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('NBA', 'Nb'),\n"," ('和', 'Caa'),\n"," ('CBA', 'FW'),\n"," ('差距', 'Na'),\n"," ('在', 'P'),\n"," ('哪裡', 'Ncd'),\n"," ('？', 'QUESTIONCATEGORY'),\n"," ('6', 'Neu'),\n"," ('張', 'Nf'),\n"," ('圖', 'VF'),\n"," ('一目瞭然', 'VH'),\n"," ('！', 'EXCLAMATIONCATEGORY')]"]},"metadata":{"tags":[]},"execution_count":21}]},{"cell_type":"markdown","metadata":{"id":"xhK9UFR_hEdR"},"source":["# Group mean vector"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"USLmUoJphEdR","executionInfo":{"status":"ok","timestamp":1608122064097,"user_tz":-480,"elapsed":1427,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"d583b6de-ef1d-4fcb-a623-5e4145c36cc9"},"source":["group_vectors = {news_class: [] for news_class in all_news_class}\n","for index, pairs in sorted(train_title_cuts.items()):\n","    vector = get_bow_vector_with_selection(pairs, word2index)\n","    news_class = train_classes[index]\n","    group_vectors[news_class].append(vector)\n","\n","group_mean_vector = {}\n","for news_class, vectors in group_vectors.items():\n","    group_mean_vector[news_class] = np.mean(vectors, axis=0)\n","group_mean_vector"],"execution_count":22,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'旅遊': array([0., 0., 0., ..., 0., 0., 0.]),\n"," '科技': array([0., 0., 0., ..., 0., 0., 0.]),\n"," '財經': array([0., 0., 0., ..., 0., 0., 0.]),\n"," '農業': array([0., 0., 0., ..., 0., 0., 0.]),\n"," '遊戲': array([0.        , 0.        , 0.        , ..., 0.00333333, 0.00333333,\n","        0.00333333]),\n"," '體育': array([0.04      , 0.01333333, 0.00333333, ..., 0.        , 0.        ,\n","        0.        ])}"]},"metadata":{"tags":[]},"execution_count":22}]},{"cell_type":"markdown","metadata":{"id":"zbSqCl6DhEdR"},"source":["# Group mean vector: 測試"]},{"cell_type":"code","metadata":{"tags":[],"id":"t8uMAj4ShEdR","executionInfo":{"status":"ok","timestamp":1608122064098,"user_tz":-480,"elapsed":1426,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}}},"source":["classification = {news_class: [] for news_class in all_news_class}\n","for index, pairs in sorted(test_title_cuts.items()):\n","    vector = get_bow_vector_with_selection(pairs, word2index)\n","    if np.sum(np.square(vector)) == 0:\n","        continue\n","\n","    max_val = -2.0\n","    max_class = None\n","    for news_class, ref_vector in group_mean_vector.items():\n","        val = cosine_similarity(ref_vector, vector)\n","        if val > max_val:\n","            max_class = news_class\n","            max_val = val\n","\n","    classification[max_class].append(index)"],"execution_count":23,"outputs":[]},{"cell_type":"code","metadata":{"tags":[],"colab":{"base_uri":"https://localhost:8080/"},"id":"KdtpBILfhEdR","executionInfo":{"status":"ok","timestamp":1608122064099,"user_tz":-480,"elapsed":1421,"user":{"displayName":"王俊煒","photoUrl":"","userId":"10865038187423164500"}},"outputId":"66d140d2-bd4f-48fc-c61a-bc1a6f91a500"},"source":["from collections import Counter\n","\n","for group, ids in classification.items():\n","    counter = Counter([test_classes[id] for id in ids])\n","    print('predict', group, ': ', counter)"],"execution_count":24,"outputs":[{"output_type":"stream","text":["predict 體育 :  Counter({'體育': 73, '遊戲': 8, '財經': 6, '旅遊': 6, '農業': 6, '科技': 3})\n","predict 財經 :  Counter({'財經': 67, '科技': 18, '體育': 6, '旅遊': 6, '遊戲': 6, '農業': 5})\n","predict 科技 :  Counter({'科技': 62, '財經': 13, '體育': 10, '農業': 6, '遊戲': 6, '旅遊': 2})\n","predict 旅遊 :  Counter({'旅遊': 70, '農業': 11, '財經': 6, '科技': 3, '遊戲': 2})\n","predict 農業 :  Counter({'農業': 70, '旅遊': 9, '科技': 6, '體育': 5, '財經': 4, '遊戲': 3})\n","predict 遊戲 :  Counter({'遊戲': 75, '科技': 7, '旅遊': 5, '體育': 4, '財經': 4, '農業': 1})\n"],"name":"stdout"}]}]}