nlp_absa_final_project.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Итоговый проект по АвтОбрЕЯ\n## Aspect-Based Sentiment Analysis\n### Команда №4: Алла Горбунова, Елизавета Клыкова, Анастасия Панасюк, Яна Шишкина","metadata":{"id":"3clrF8JeWPSW"}},{"cell_type":"markdown","source":"## Импорты и подготовка данных","metadata":{"id":"HsHcd51IYATY"}},{"cell_type":"code","source":"!pip install stanza\n!pip install transformers\n!pip install fasttext","metadata":{"id":"AE45LLLuWNDg","outputId":"cb37aaf8-588b-497c-e834-ee7e6bf84063","execution":{"iopub.status.busy":"2021-12-27T12:18:29.190802Z","iopub.execute_input":"2021-12-27T12:18:29.191179Z","iopub.status.idle":"2021-12-27T12:18:53.269177Z","shell.execute_reply.started":"2021-12-27T12:18:29.191085Z","shell.execute_reply":"2021-12-27T12:18:53.268367Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Collecting stanza\n  Downloading stanza-1.3.0-py3-none-any.whl (432 kB)\n     |████████████████████████████████| 432 kB 4.4 MB/s            \n\u001b[?25hRequirement already satisfied: torch>=1.3.0 in /opt/conda/lib/python3.7/site-packages (from stanza) (1.9.1)\nRequirement already satisfied: protobuf in /opt/conda/lib/python3.7/site-packages (from stanza) (3.19.1)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from stanza) (2.25.1)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from stanza) (1.19.5)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from stanza) (1.16.0)\nRequirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (from stanza) (4.62.3)\nRequirement already satisfied: emoji in /opt/conda/lib/python3.7/site-packages (from stanza) (1.6.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch>=1.3.0->stanza) (3.10.0.2)\nRequirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->stanza) (4.0.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->stanza) (2021.10.8)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->stanza) (1.26.7)\nRequirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->stanza) (2.10)\nInstalling collected packages: stanza\nSuccessfully installed stanza-1.3.0\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\nRequirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.12.5)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.3.2)\nRequirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.10.3)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (21.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.1.2)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (4.8.2)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.25.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (6.0)\nRequirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.46)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.62.3)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2021.11.10)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.19.5)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\nRequirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=20.0->transformers) (3.0.6)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.6.0)\nRequirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.10)\nRequirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (4.0.0)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2021.10.8)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.7)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.1.0)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (8.0.3)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.16.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\nRequirement already satisfied: fasttext in /opt/conda/lib/python3.7/site-packages (0.9.2)\nRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from fasttext) (1.19.5)\nRequirement already satisfied: pybind11>=2.2 in /opt/conda/lib/python3.7/site-packages (from fasttext) (2.8.1)\nRequirement already satisfied: setuptools>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from fasttext) (59.1.1)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n","output_type":"stream"}]},{"cell_type":"code","source":"import re\n\nimport random\nimport numpy as np\nimport pandas as pd\n\nfrom collections import defaultdict, Counter\nfrom string import punctuation\nfrom tqdm.auto import tqdm\nfrom copy import copy\n\nimport stanza\n\nimport torch\nfrom transformers import BertTokenizerFast, BertConfig, \\\n    BertForTokenClassification, AutoModelForSequenceClassification\n\nimport fasttext\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.naive_bayes import GaussianNB","metadata":{"id":"NOGNuLqXWefp","execution":{"iopub.status.busy":"2021-12-27T12:18:53.272917Z","iopub.execute_input":"2021-12-27T12:18:53.273195Z","iopub.status.idle":"2021-12-27T12:19:00.602542Z","shell.execute_reply.started":"2021-12-27T12:18:53.273162Z","shell.execute_reply":"2021-12-27T12:19:00.601790Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"seed = 200\nrandom.seed(seed)\nnp.random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:19:00.609526Z","iopub.execute_input":"2021-12-27T12:19:00.610007Z","iopub.status.idle":"2021-12-27T12:19:00.616042Z","shell.execute_reply.started":"2021-12-27T12:19:00.609971Z","shell.execute_reply":"2021-12-27T12:19:00.613162Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"!wget https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/dev_reviews.txt","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:19:00.617275Z","iopub.execute_input":"2021-12-27T12:19:00.617737Z","iopub.status.idle":"2021-12-27T12:19:01.629765Z","shell.execute_reply.started":"2021-12-27T12:19:00.617699Z","shell.execute_reply":"2021-12-27T12:19:01.629025Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"--2021-12-27 12:19:01--  https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/dev_reviews.txt\nResolving github.com (github.com)... 140.82.113.4\nConnecting to github.com (github.com)|140.82.113.4|:443... connected.\nHTTP request sent, awaiting response... 302 Found\nLocation: https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/dev_reviews.txt [following]\n--2021-12-27 12:19:01--  https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/dev_reviews.txt\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.111.133, 185.199.108.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 110515 (108K) [text/plain]\nSaving to: ‘dev_reviews.txt’\n\ndev_reviews.txt     100%[===================>] 107.92K  --.-KB/s    in 0.02s   \n\n2021-12-27 12:19:01 (5.21 MB/s) - ‘dev_reviews.txt’ saved [110515/110515]\n\n","output_type":"stream"}]},{"cell_type":"code","source":"reviews = {}\nwith open('dev_reviews.txt') as f:\n    for line in f:\n        line = line.rstrip('\\r\\n').split('\\t')\n        reviews[int(line[0])] = line[1]","metadata":{"id":"F35ufWZ0Wnsv","execution":{"iopub.status.busy":"2021-12-27T12:19:01.631580Z","iopub.execute_input":"2021-12-27T12:19:01.632154Z","iopub.status.idle":"2021-12-27T12:19:01.640338Z","shell.execute_reply.started":"2021-12-27T12:19:01.632113Z","shell.execute_reply":"2021-12-27T12:19:01.639600Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"len(reviews)","metadata":{"id":"3Q7ld3z0XpvF","outputId":"f39c1e74-d359-4b0c-80f1-7a389d27af5c","execution":{"iopub.status.busy":"2021-12-27T12:19:01.641643Z","iopub.execute_input":"2021-12-27T12:19:01.641944Z","iopub.status.idle":"2021-12-27T12:19:01.653852Z","shell.execute_reply.started":"2021-12-27T12:19:01.641909Z","shell.execute_reply":"2021-12-27T12:19:01.653028Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":"71"},"metadata":{}}]},{"cell_type":"markdown","source":"## Этап 1: парсим тексты Станзой","metadata":{"id":"jH3__J06YHQP"}},{"cell_type":"code","source":"stanza.download('ru')\nnlp = stanza.Pipeline('ru', processors='tokenize,pos,depparse,lemma')","metadata":{"id":"yBGtmXP_XuOZ","outputId":"f457e8de-d015-4d3c-8d0e-38e8d55e4ff8","execution":{"iopub.status.busy":"2021-12-27T12:19:01.656288Z","iopub.execute_input":"2021-12-27T12:19:01.656952Z","iopub.status.idle":"2021-12-27T12:19:34.189344Z","shell.execute_reply.started":"2021-12-27T12:19:01.656916Z","shell.execute_reply":"2021-12-27T12:19:34.188595Z"},"trusted":true},"execution_count":7,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.3.0.json:   0%|   …","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8080dcafdde64cd991b4a6b22e76003d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading https://huggingface.co/stanfordnlp/stanza-ru/resolve/v1.3.0/models/default.zip:   0%|          | 0…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ca7c985899794b76b5ef92563dc5795e"}},"metadata":{}}]},{"cell_type":"code","source":"def parse_with_stanza(nlp, reviews: dict):\n    parsed_reviews = {}\n    for text_idx, text in tqdm(list(reviews.items())):\n        doc = nlp(text)\n        doc_dict = {}\n        for sent_num, sentence in enumerate(doc.sentences):\n            sent_dict = {}\n            for tok_num, token in enumerate(sentence.tokens):\n                tok_dict = {'id': token.id[0] - 1,\n                            'text': token.text,\n                            'start': token.start_char,\n                            'end': token.end_char,\n                            'pos': token.words[0].upos,\n                            'head': token.words[0].head - 1,\n                            'deprel': token.words[0].deprel,\n                            'sent_index': sent_num,\n                            'text_index': text_idx}\n                sent_dict[tok_num] = tok_dict\n            doc_dict[sent_num] = sent_dict\n        parsed_reviews[text_idx] = doc_dict\n    return parsed_reviews","metadata":{"id":"CIHiwsiTYken","execution":{"iopub.status.busy":"2021-12-27T12:19:34.190609Z","iopub.execute_input":"2021-12-27T12:19:34.190874Z","iopub.status.idle":"2021-12-27T12:19:34.199272Z","shell.execute_reply.started":"2021-12-27T12:19:34.190840Z","shell.execute_reply":"2021-12-27T12:19:34.198029Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"test_parsed = parse_with_stanza(nlp, reviews)","metadata":{"id":"Orne5tJNb3e_","outputId":"8188a3bb-7327-4b09-c225-31e650d74035","execution":{"iopub.status.busy":"2021-12-27T12:19:34.203136Z","iopub.execute_input":"2021-12-27T12:19:34.203669Z","iopub.status.idle":"2021-12-27T12:19:45.403411Z","shell.execute_reply.started":"2021-12-27T12:19:34.203635Z","shell.execute_reply":"2021-12-27T12:19:45.402711Z"},"trusted":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/71 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"56bccf76828c469b8932915e9552f46e"}},"metadata":{}}]},{"cell_type":"code","source":"test_parsed[13823][0][0]","metadata":{"id":"vZjlCXoLcENq","outputId":"06d3e1d0-8dfd-4a8d-8b5a-582c3152799c","execution":{"iopub.status.busy":"2021-12-27T12:19:45.404535Z","iopub.execute_input":"2021-12-27T12:19:45.405171Z","iopub.status.idle":"2021-12-27T12:19:45.411911Z","shell.execute_reply.started":"2021-12-27T12:19:45.405131Z","shell.execute_reply":"2021-12-27T12:19:45.411291Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"{'id': 0,\n 'text': 'Зашли',\n 'start': 0,\n 'end': 5,\n 'pos': 'VERB',\n 'head': -1,\n 'deprel': 'root',\n 'sent_index': 0,\n 'text_index': 13823}"},"metadata":{}}]},{"cell_type":"markdown","source":"## Этап 2: размечаем аспекты Бертом\nОбучение модели [здесь](https://colab.research.google.com/drive/1e37Ek7kQjaOvyuutef2AHx57XrfTI7BJ?usp=sharing); в этой тетрадке будем использовать готовую. Сама модель лежит [тут](https://drive.google.com/drive/folders/1BIz1jpXK4GLQYHFL3-vy7NXuApIEhMbW?usp=sharing).","metadata":{"id":"bu0jLpeOcjBN"}},{"cell_type":"code","source":"# from google.colab import drive\n# drive.mount('/content/gdrive')","metadata":{"id":"j__ALOj4cra0","outputId":"38c36d0c-d2d2-4aa3-b66a-db9b2b0fc62d","execution":{"iopub.status.busy":"2021-12-27T12:19:45.413169Z","iopub.execute_input":"2021-12-27T12:19:45.413896Z","iopub.status.idle":"2021-12-27T12:19:45.421467Z","shell.execute_reply.started":"2021-12-27T12:19:45.413858Z","shell.execute_reply":"2021-12-27T12:19:45.420666Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"id":"KWzn8rB9dFzV","outputId":"9944b038-4462-4cae-fd36-18f4707bd629","execution":{"iopub.status.busy":"2021-12-27T12:21:40.215353Z","iopub.execute_input":"2021-12-27T12:21:40.215963Z","iopub.status.idle":"2021-12-27T12:21:40.223147Z","shell.execute_reply.started":"2021-12-27T12:21:40.215916Z","shell.execute_reply":"2021-12-27T12:21:40.222347Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"'cuda'"},"metadata":{}}]},{"cell_type":"code","source":"tokenizer = BertTokenizerFast.from_pretrained('DeepPavlov/rubert-base-cased')\n\naspect_model = BertForTokenClassification.from_pretrained(\n    'DeepPavlov/rubert-base-cased', num_labels=3)\naspect_model.to(device)\n\n# поменять пути, если в Колабе\naspect_model_path = '../input/aspect-model/aspect_model/pytorch_model.bin'\naspect_model.load_state_dict(torch.load(aspect_model_path))","metadata":{"id":"GhGc-K6Oc-0b","outputId":"f9b12daf-e40c-40c8-929b-272ef356a663","execution":{"iopub.status.busy":"2021-12-27T12:21:40.225050Z","iopub.execute_input":"2021-12-27T12:21:40.225602Z","iopub.status.idle":"2021-12-27T12:21:53.643256Z","shell.execute_reply.started":"2021-12-27T12:21:40.225563Z","shell.execute_reply":"2021-12-27T12:21:53.642580Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at DeepPavlov/rubert-base-cased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of BertForTokenClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"<All keys matched successfully>"},"metadata":{}}]},{"cell_type":"code","source":"def bio_sentence(sentence, tokenizer, model, device, ids_to_labels):\n    # подаем токены из Станзы\n    input_tokens = [tok['text'] for tok in list(sentence.values())]\n    inputs = tokenizer(input_tokens,\n                       is_split_into_words=True,\n                       return_offsets_mapping=True,\n                       padding='max_length',\n                       truncation=True,\n                       max_length=512,\n                       return_tensors='pt')\n\n    ids = inputs['input_ids'].to(device)\n    mask = inputs['attention_mask'].to(device)\n    outputs = model(ids, attention_mask=mask)\n    logits = outputs[0]\n\n    active_logits = logits.view(-1, model.num_labels)\n    flattened_predictions = torch.argmax(active_logits, axis=1)\n\n    tokens = tokenizer.convert_ids_to_tokens(ids.squeeze().tolist())\n    token_predictions = [ids_to_labels[i]\n                         for i in flattened_predictions.cpu().numpy()]\n    wp_preds = list(zip(tokens, token_predictions))\n\n    sent_prediction = []\n    for token_pred, mapping in zip(wp_preds,\n                                   inputs[\"offset_mapping\"].squeeze().tolist()):\n        if mapping[0] == 0 and mapping[1] != 0:\n            sent_prediction.append(token_pred[1])\n        else:\n            continue\n\n    output = copy(sentence)\n    for i, tag in enumerate(sent_prediction):\n        output[i]['bio'] = tag\n\n    return output","metadata":{"id":"Wz_Vo1Oydqw7","execution":{"iopub.status.busy":"2021-12-27T12:21:53.644438Z","iopub.execute_input":"2021-12-27T12:21:53.644678Z","iopub.status.idle":"2021-12-27T12:21:53.656756Z","shell.execute_reply.started":"2021-12-27T12:21:53.644641Z","shell.execute_reply":"2021-12-27T12:21:53.656124Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"def bio_corpus(reviews, tokenizer, model, device):\n    bio_tags = ['O', 'B', 'I']\n    ids_to_labels = {v: k for v, k in enumerate(bio_tags)}\n\n    tagged_reviews = {}\n    for idx, review in tqdm(list(reviews.items())):\n        tagged_text = {}\n        for i, sentence in list(review.items()):\n            tagged_sent = bio_sentence(sentence, tokenizer, model,\n                                         device, ids_to_labels)\n            tagged_text[i] = tagged_sent\n        tagged_reviews[idx] = tagged_text\n\n    return tagged_reviews","metadata":{"id":"BS5-nImQgVrG","execution":{"iopub.status.busy":"2021-12-27T12:21:53.659023Z","iopub.execute_input":"2021-12-27T12:21:53.659317Z","iopub.status.idle":"2021-12-27T12:21:53.667511Z","shell.execute_reply.started":"2021-12-27T12:21:53.659278Z","shell.execute_reply":"2021-12-27T12:21:53.666753Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"test_with_bio = bio_corpus(test_parsed, tokenizer, aspect_model, device)","metadata":{"id":"s2ICsXCZj1Bn","outputId":"ce51cab7-945c-4b9b-91b3-0e4a31e6c9a0","execution":{"iopub.status.busy":"2021-12-27T12:21:53.668712Z","iopub.execute_input":"2021-12-27T12:21:53.668936Z","iopub.status.idle":"2021-12-27T12:22:12.697630Z","shell.execute_reply.started":"2021-12-27T12:21:53.668899Z","shell.execute_reply":"2021-12-27T12:22:12.696942Z"},"trusted":true},"execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/71 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5ea8ea3272a9459592531e14339de175"}},"metadata":{}}]},{"cell_type":"code","source":"test_with_bio[13823][0][0]","metadata":{"id":"O6R8PmgxkKAU","outputId":"c7b4e9d8-a4ee-4726-85f0-4fc6611a5a82","execution":{"iopub.status.busy":"2021-12-27T12:22:12.698936Z","iopub.execute_input":"2021-12-27T12:22:12.699321Z","iopub.status.idle":"2021-12-27T12:22:12.705725Z","shell.execute_reply.started":"2021-12-27T12:22:12.699278Z","shell.execute_reply":"2021-12-27T12:22:12.704942Z"},"trusted":true},"execution_count":19,"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"{'id': 0,\n 'text': 'Зашли',\n 'start': 0,\n 'end': 5,\n 'pos': 'VERB',\n 'head': -1,\n 'deprel': 'root',\n 'sent_index': 0,\n 'text_index': 13823,\n 'bio': 'O'}"},"metadata":{}}]},{"cell_type":"markdown","source":"## Этап 3: оцениваем тональность","metadata":{"id":"ccZ_5GJRl27X"}},{"cell_type":"markdown","source":"### 3.1. Fine-Tuned BERT\nОбучение модели [здесь](https://colab.research.google.com/drive/1uqYLiWqBpdiDZKCBMFMiLFA88lJitTa8?usp=sharing) (это не та же, что в предыдущем пункте); в этой тетрадке будем использовать готовую. Сама модель лежит [тут](https://drive.google.com/drive/folders/191PmR-fVBdkCs3Xxx4n7KVTeBnVJhY8V?usp=sharing), в той же тетрадке можно посмотреть на ее оценку.\n\nПри таком подходе нам не нужно думать о выделении однословных или многословных аспектов: модель приписывает сентимент-тег каждому токену.","metadata":{"id":"_f2q-wpIrHiB"}},{"cell_type":"code","source":"tokenizer = BertTokenizerFast.from_pretrained('DeepPavlov/rubert-base-cased')\n\nsentim_model = BertForTokenClassification.from_pretrained(\n    'DeepPavlov/rubert-base-cased', num_labels=4)\nsentim_model.to(device)\n\nsentim_model_path = '../input/aspect-sentiment-model/aspect_sentiment_model/pytorch_model.bin'\nsentim_model.load_state_dict(torch.load(sentim_model_path))","metadata":{"id":"neYUpgkFl2dY","outputId":"fbdfab59-b0ab-4004-f91c-02904aeacba1","execution":{"iopub.status.busy":"2021-12-27T12:22:12.706963Z","iopub.execute_input":"2021-12-27T12:22:12.707385Z","iopub.status.idle":"2021-12-27T12:22:24.766125Z","shell.execute_reply.started":"2021-12-27T12:22:12.707346Z","shell.execute_reply":"2021-12-27T12:22:24.765393Z"},"trusted":true},"execution_count":20,"outputs":[{"name":"stderr","text":"Some weights of the model checkpoint at DeepPavlov/rubert-base-cased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight']\n- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\nSome weights of BertForTokenClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"<All keys matched successfully>"},"metadata":{}}]},{"cell_type":"code","source":"def sentiment_sentence(sentence, tokenizer, model, device, ids_to_labels):\n    # подаем токены из Станзы\n    input_tokens = [tok['text'] for tok in list(sentence.values())]\n    inputs = tokenizer(input_tokens,\n                       is_split_into_words=True,\n                       return_offsets_mapping=True,\n                       padding='max_length',\n                       truncation=True,\n                       max_length=512,\n                       return_tensors='pt')\n\n    ids = inputs['input_ids'].to(device)\n    mask = inputs['attention_mask'].to(device)\n    outputs = model(ids, attention_mask=mask)\n    logits = outputs[0]\n\n    active_logits = logits.view(-1, model.num_labels)\n    flattened_predictions = torch.argmax(active_logits, axis=1)\n    tokens = tokenizer.convert_ids_to_tokens(ids.squeeze().tolist())\n    token_predictions = [ids_to_labels[i]\n                         for i in flattened_predictions.cpu().numpy()]\n    wp_preds = list(zip(tokens, token_predictions))\n\n    sent_prediction = []\n    for token_pred, mapping in zip(wp_preds,\n                                   inputs['offset_mapping'].squeeze().tolist()):\n        if mapping[0] == 0 and mapping[1] != 0:\n            sent_prediction.append(token_pred[1])\n        else:\n            continue\n\n    output = copy(sentence)\n    for i, tag in enumerate(sent_prediction):\n        output[i]['sentiment1'] = tag\n\n    return output","metadata":{"id":"XffSNK0bn8Pu","execution":{"iopub.status.busy":"2021-12-27T12:22:24.767353Z","iopub.execute_input":"2021-12-27T12:22:24.767770Z","iopub.status.idle":"2021-12-27T12:22:24.778359Z","shell.execute_reply.started":"2021-12-27T12:22:24.767729Z","shell.execute_reply":"2021-12-27T12:22:24.777603Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"def sentiment_corpus(reviews, tokenizer, model, device):\n    sent_tags = ['neutral', 'positive', 'negative', 'both']\n    ids_to_labels = {v: k for v, k in enumerate(sent_tags)}\n    # дизайнерское решение, объясняется в тетрадке с обучением\n    ids_to_labels[-100] = 'O'\n\n    tagged_reviews = {}\n    for idx, review in tqdm(list(reviews.items())):\n        tagged_text = {}\n        for i, sentence in list(review.items()):\n            tagged_sent = sentiment_sentence(sentence, tokenizer, model,\n                                             device, ids_to_labels)\n            tagged_text[i] = tagged_sent\n        tagged_reviews[idx] = tagged_text\n\n    return tagged_reviews","metadata":{"id":"mfYOMDrcpVIO","execution":{"iopub.status.busy":"2021-12-27T12:22:24.779499Z","iopub.execute_input":"2021-12-27T12:22:24.779807Z","iopub.status.idle":"2021-12-27T12:22:24.793729Z","shell.execute_reply.started":"2021-12-27T12:22:24.779769Z","shell.execute_reply":"2021-12-27T12:22:24.792935Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"test_with_sentiment = sentiment_corpus(test_with_bio, tokenizer,\n                                       sentim_model, device)","metadata":{"id":"XQKtnKRPpx8V","outputId":"7abc0d86-ffbc-4aec-8c86-1626a6ef554b","execution":{"iopub.status.busy":"2021-12-27T12:22:24.797778Z","iopub.execute_input":"2021-12-27T12:22:24.797988Z","iopub.status.idle":"2021-12-27T12:22:43.833544Z","shell.execute_reply.started":"2021-12-27T12:22:24.797957Z","shell.execute_reply":"2021-12-27T12:22:43.832785Z"},"trusted":true},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/71 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"529dc0b2af0a4c02a30e37e47139e19b"}},"metadata":{}}]},{"cell_type":"code","source":"test_with_sentiment[13823][0][0]","metadata":{"id":"oecObbgqq8SK","outputId":"f33fa18b-774b-4605-8b0b-73301d327bb7","execution":{"iopub.status.busy":"2021-12-27T12:22:43.834742Z","iopub.execute_input":"2021-12-27T12:22:43.835485Z","iopub.status.idle":"2021-12-27T12:22:43.842377Z","shell.execute_reply.started":"2021-12-27T12:22:43.835444Z","shell.execute_reply":"2021-12-27T12:22:43.841543Z"},"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"{'id': 0,\n 'text': 'Зашли',\n 'start': 0,\n 'end': 5,\n 'pos': 'VERB',\n 'head': -1,\n 'deprel': 'root',\n 'sent_index': 0,\n 'text_index': 13823,\n 'bio': 'O',\n 'sentiment1': 'neutral'}"},"metadata":{}}]},{"cell_type":"markdown","source":"### 3.2. Выделение аспектов\nДля каждого токена мы получили указание на то, является ли токен частью аспекта, и его тональность. Теперь нам нужно собрать части аспектов в целые сущности.","metadata":{"id":"-gxBMMxdkzJ6"}},{"cell_type":"code","source":"def get_sentence_aspects(one_sent):\n\n    aspects = []\n    one_aspect = []\n    prev = None\n\n    for i, token in list(one_sent.items()):\n\n        # если токен -- начало сущности, запоминаем\n        if token['bio'] == 'B' and prev != 'B' and prev != 'I':\n            one_aspect.append(token)\n\n        # если токен -- начало нового аспекта, запоминаем предыдущий и новый\n        elif token['bio'] == 'B' and (prev == 'B' or prev == 'I'):\n            aspects.append(one_aspect)\n            one_aspect = [token]\n\n        # если токен -- продолжение сущности, запоминаем\n        # сюда же попадут сущности, по ошибке не начинающиеся с B\n        elif token['bio'] == 'I':\n            one_aspect.append(token)\n\n        # если сущность закончилась, запоминаем и обнуляем аспект\n        elif token['bio'] == 'O' and (prev == 'I' or prev == 'B'):\n            aspects.append(one_aspect)\n            one_aspect = []\n\n        # запоминаем, какой тег только что видели\n        prev = token['bio']\n\n    return aspects","metadata":{"id":"ckXbrCJ0kgvX","execution":{"iopub.status.busy":"2021-12-27T12:22:43.843707Z","iopub.execute_input":"2021-12-27T12:22:43.844684Z","iopub.status.idle":"2021-12-27T12:22:43.854849Z","shell.execute_reply.started":"2021-12-27T12:22:43.844644Z","shell.execute_reply":"2021-12-27T12:22:43.854139Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"markdown","source":"Мы научились получать аспекты на уровне одного предложения. Теперь приведем их в нужный формат: соединим через пробел, припишем тональность, индекс начала и конца и т.д. Тональность аспекта выбираем как тональность первого слова.","metadata":{"id":"PHim7jFr1yji"}},{"cell_type":"code","source":"all_aspects = []  # здесь все аспекты корпуса, чтобы было удобнее смотреть\ntext_aspects = []  # здесь аспекты, разделенные по текстам\nfor idx, text in tqdm(list(test_with_sentiment.items())):\n    one_text = []\n    for i, sent in list(text.items()):\n        one_sent = []\n        sent_aspects = [asp for asp in get_sentence_aspects(sent) if asp]\n        for aspect in sent_aspects:\n            clean_aspect = {'asp_idx': aspect[0]['text_index'],\n                            'asp_text': ' '.join([pt['text'] for pt in aspect]),\n                            'asp_start': aspect[0]['start'],\n                            'asp_end': aspect[-1]['end'],\n                            'sentiment1': aspect[0]['sentiment1']}\n            all_aspects.append(clean_aspect)\n            one_sent.append(clean_aspect)\n        one_text.append(one_sent)\n    text_aspects.append(one_text)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:22:43.856173Z","iopub.execute_input":"2021-12-27T12:22:43.856677Z","iopub.status.idle":"2021-12-27T12:22:43.916272Z","shell.execute_reply.started":"2021-12-27T12:22:43.856568Z","shell.execute_reply":"2021-12-27T12:22:43.915538Z"},"trusted":true},"execution_count":26,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/71 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9b1cd39bf8cf479186dffd63e3e3c583"}},"metadata":{}}]},{"cell_type":"markdown","source":"Ниже еще один способ выбирать тональность -- с подсчетом тональностей всех слов и выбором наиболее частотной. На второй задаче этот подход показал себя чуть хуже, и мы остановились на предыдущем.","metadata":{}},{"cell_type":"code","source":"# all_aspects = []  # здесь все аспекты корпуса, чтобы было удобнее смотреть\n# text_aspects = []  # здесь аспекты, разделенные по текстам\n# for idx, text in tqdm(list(test_with_sentiment.items())):\n#     one_text = []\n#     for i, sent in list(text.items()):\n#         sent_aspects = [asp for asp in get_sentence_aspects(sent) if asp]\n#         for aspect in sent_aspects:\n#             clean_aspect = {'asp_idx': aspect[0]['text_index'],\n#                             'asp_text': ' '.join([pt['text'] for pt in aspect]),\n#                             'asp_start': aspect[0]['start'],\n#                             'asp_end': aspect[-1]['end']}\n#             tones = Counter([pt['sentiment1'] for pt in aspect]).most_common()\n#             if len(tones) == 1:\n#                 clean_aspect['sentiment1'] = tones[0][0]\n#             else:\n#                 if tones[0][1] > tones[1][1]:\n#                     clean_aspect['sentiment1'] = tones[0][0]\n#                 else:\n#                     clean_aspect['sentiment1'] = 'both'\n#             all_aspects.append(clean_aspect)\n#             one_text.append(clean_aspect)\n#     text_aspects.append(one_text)","metadata":{"id":"UpHdKRS4xQTA","outputId":"e208f563-6df6-437e-edad-975438837b41","execution":{"iopub.status.busy":"2021-12-27T12:22:43.917619Z","iopub.execute_input":"2021-12-27T12:22:43.917865Z","iopub.status.idle":"2021-12-27T12:22:43.922459Z","shell.execute_reply.started":"2021-12-27T12:22:43.917831Z","shell.execute_reply":"2021-12-27T12:22:43.921576Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"code","source":"len(all_aspects)","metadata":{"id":"zZPmO0mV5dm6","outputId":"3eba5058-38d6-4189-a3e9-6d38f552a7cc","execution":{"iopub.status.busy":"2021-12-27T12:22:43.924085Z","iopub.execute_input":"2021-12-27T12:22:43.924359Z","iopub.status.idle":"2021-12-27T12:22:43.937653Z","shell.execute_reply.started":"2021-12-27T12:22:43.924324Z","shell.execute_reply":"2021-12-27T12:22:43.936957Z"},"trusted":true},"execution_count":28,"outputs":[{"execution_count":28,"output_type":"execute_result","data":{"text/plain":"1180"},"metadata":{}}]},{"cell_type":"markdown","source":"Посмотрим, сколько аспектов вообще должно быть в этих данных.","metadata":{"id":"HJyPM-mGBJ50"}},{"cell_type":"code","source":"!wget https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/dev_aspects.txt","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:27:12.692527Z","iopub.execute_input":"2021-12-27T12:27:12.692848Z","iopub.status.idle":"2021-12-27T12:27:13.677619Z","shell.execute_reply.started":"2021-12-27T12:27:12.692815Z","shell.execute_reply":"2021-12-27T12:27:13.676783Z"},"trusted":true},"execution_count":31,"outputs":[{"name":"stdout","text":"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\nTo disable this warning, you can either:\n\t- Avoid using `tokenizers` before the fork if possible\n\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n--2021-12-27 12:27:13--  https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/dev_aspects.txt\nResolving github.com (github.com)... 140.82.114.4\nConnecting to github.com (github.com)|140.82.114.4|:443... connected.\nHTTP request sent, awaiting response... 302 Found\nLocation: https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/dev_aspects.txt [following]\n--2021-12-27 12:27:13--  https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/dev_aspects.txt\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 57508 (56K) [text/plain]\nSaving to: ‘dev_aspects.txt’\n\ndev_aspects.txt     100%[===================>]  56.16K  --.-KB/s    in 0.01s   \n\n2021-12-27 12:27:13 (5.05 MB/s) - ‘dev_aspects.txt’ saved [57508/57508]\n\n","output_type":"stream"}]},{"cell_type":"code","source":"gold_aspects = []\nkeys = ('asp_idx', 'category', 'asp_text', 'asp_start', 'asp_end', 'sentiment1')\n\nwith open('dev_aspects.txt') as f:\n    for line in f:\n        line = line.rstrip('\\r\\n').split('\\t')\n        line_dict = {}\n        for i, key in enumerate(keys):\n            line_dict[key] = line[i]\n        gold_aspects.append(line_dict)","metadata":{"id":"03PS5-23_ccF","execution":{"iopub.status.busy":"2021-12-27T12:27:13.679426Z","iopub.execute_input":"2021-12-27T12:27:13.679710Z","iopub.status.idle":"2021-12-27T12:27:13.692959Z","shell.execute_reply.started":"2021-12-27T12:27:13.679682Z","shell.execute_reply":"2021-12-27T12:27:13.692156Z"},"trusted":true},"execution_count":32,"outputs":[]},{"cell_type":"code","source":"len(gold_aspects)","metadata":{"id":"qVXaRAtI_hpo","outputId":"b0e46c3d-2887-4389-9be5-c7fe798c337d","execution":{"iopub.status.busy":"2021-12-27T12:27:13.695190Z","iopub.execute_input":"2021-12-27T12:27:13.695534Z","iopub.status.idle":"2021-12-27T12:27:13.704623Z","shell.execute_reply.started":"2021-12-27T12:27:13.695504Z","shell.execute_reply":"2021-12-27T12:27:13.703739Z"},"trusted":true},"execution_count":33,"outputs":[{"execution_count":33,"output_type":"execute_result","data":{"text/plain":"1190"},"metadata":{}}]},{"cell_type":"markdown","source":"## Этап 4: определяем категории\nДля определения категорий будем использовать классификацию. Попробуем несколько алгоритмов.","metadata":{"id":"cyhKmVxoBgtg"}},{"cell_type":"markdown","source":"#### Подготовка данных\nНам нужно взять те же данные, на которых обучались предыдущие две модели, и таким же образом поделить их на обучающие и тестовые.","metadata":{"id":"Svldm-_BDqqc"}},{"cell_type":"code","source":"!wget https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/train_aspects.txt","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:27:13.706193Z","iopub.execute_input":"2021-12-27T12:27:13.707041Z","iopub.status.idle":"2021-12-27T12:27:14.698593Z","shell.execute_reply.started":"2021-12-27T12:27:13.707003Z","shell.execute_reply":"2021-12-27T12:27:14.697747Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\nTo disable this warning, you can either:\n\t- Avoid using `tokenizers` before the fork if possible\n\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n--2021-12-27 12:27:14--  https://github.com/named-entity/hse-nlp/raw/master/4th_year/Project/train_aspects.txt\nResolving github.com (github.com)... 140.82.114.4\nConnecting to github.com (github.com)|140.82.114.4|:443... connected.\nHTTP request sent, awaiting response... 302 Found\nLocation: https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/train_aspects.txt [following]\n--2021-12-27 12:27:14--  https://raw.githubusercontent.com/named-entity/hse-nlp/master/4th_year/Project/train_aspects.txt\nResolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.110.133, 185.199.108.133, ...\nConnecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 228391 (223K) [text/plain]\nSaving to: ‘train_aspects.txt’\n\ntrain_aspects.txt   100%[===================>] 223.04K  --.-KB/s    in 0.03s   \n\n2021-12-27 12:27:14 (8.68 MB/s) - ‘train_aspects.txt’ saved [228391/228391]\n\n","output_type":"stream"}]},{"cell_type":"code","source":"asp_df = pd.read_csv('train_aspects.txt', sep='\\t', usecols=[0, 1, 2],\n                     names=['text_idx', 'category', 'aspect'], header=None)\nasp_df","metadata":{"id":"S66dJ-4uBwM8","outputId":"b7c255f6-82a4-4724-acb6-5f555b287db1","execution":{"iopub.status.busy":"2021-12-27T12:27:14.701743Z","iopub.execute_input":"2021-12-27T12:27:14.702218Z","iopub.status.idle":"2021-12-27T12:27:14.733764Z","shell.execute_reply.started":"2021-12-27T12:27:14.702175Z","shell.execute_reply":"2021-12-27T12:27:14.733067Z"},"trusted":true},"execution_count":35,"outputs":[{"execution_count":35,"output_type":"execute_result","data":{"text/plain":"      text_idx category              aspect\n0         3976    Whole           ресторане\n1         3976    Whole          ресторанах\n2         3976    Whole           ресторане\n3         3976  Service  Столик бронировали\n4         3976  Service       администратор\n...        ...      ...                 ...\n4758     16630  Service        обслуживание\n4759     16630     Food                 Еда\n4760     16630  Service           персоналу\n4761     16630    Whole            ресторан\n4762     16630    Whole               место\n\n[4763 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text_idx</th>\n      <th>category</th>\n      <th>aspect</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторане</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторанах</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторане</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>Столик бронировали</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>администратор</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4758</th>\n      <td>16630</td>\n      <td>Service</td>\n      <td>обслуживание</td>\n    </tr>\n    <tr>\n      <th>4759</th>\n      <td>16630</td>\n      <td>Food</td>\n      <td>Еда</td>\n    </tr>\n    <tr>\n      <th>4760</th>\n      <td>16630</td>\n      <td>Service</td>\n      <td>персоналу</td>\n    </tr>\n    <tr>\n      <th>4761</th>\n      <td>16630</td>\n      <td>Whole</td>\n      <td>ресторан</td>\n    </tr>\n    <tr>\n      <th>4762</th>\n      <td>16630</td>\n      <td>Whole</td>\n      <td>место</td>\n    </tr>\n  </tbody>\n</table>\n<p>4763 rows × 3 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"mapper = {topic: t for t, topic in enumerate(asp_df['category'].unique())}\nprint(mapper)\n\nasp_df['target'] = asp_df['category'].map(mapper)\nasp_df.head(5)","metadata":{"id":"ZyZpEj4WCBZc","outputId":"ca9e95fc-d8d8-49a3-bff6-e0f3778bdbda","execution":{"iopub.status.busy":"2021-12-27T12:27:14.735375Z","iopub.execute_input":"2021-12-27T12:27:14.735925Z","iopub.status.idle":"2021-12-27T12:27:14.754424Z","shell.execute_reply.started":"2021-12-27T12:27:14.735881Z","shell.execute_reply":"2021-12-27T12:27:14.753674Z"},"trusted":true},"execution_count":36,"outputs":[{"name":"stdout","text":"{'Whole': 0, 'Service': 1, 'Food': 2, 'Interior': 3, 'Price': 4}\n","output_type":"stream"},{"execution_count":36,"output_type":"execute_result","data":{"text/plain":"   text_idx category              aspect  target\n0      3976    Whole           ресторане       0\n1      3976    Whole          ресторанах       0\n2      3976    Whole           ресторане       0\n3      3976  Service  Столик бронировали       1\n4      3976  Service       администратор       1","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text_idx</th>\n      <th>category</th>\n      <th>aspect</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторане</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторанах</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>ресторане</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>Столик бронировали</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>администратор</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"def process(text):\n    splitted = re.split(' |/', text.lower())\n    tokens = [re.sub('[^а-яёa-z0-9-]', '', word) for word in splitted]\n    return tokens","metadata":{"id":"rjgkUt-BDWkZ","execution":{"iopub.status.busy":"2021-12-27T12:27:14.756286Z","iopub.execute_input":"2021-12-27T12:27:14.756874Z","iopub.status.idle":"2021-12-27T12:27:14.764341Z","shell.execute_reply.started":"2021-12-27T12:27:14.756779Z","shell.execute_reply":"2021-12-27T12:27:14.763622Z"},"trusted":true},"execution_count":37,"outputs":[]},{"cell_type":"code","source":"asp_df['aspect'] = asp_df['aspect'].apply(process)\nasp_df.head(5)","metadata":{"id":"fsvyNQ_iDX4C","outputId":"778976b7-c5f2-472a-ed79-f18a85d2a0ca","execution":{"iopub.status.busy":"2021-12-27T12:27:14.766206Z","iopub.execute_input":"2021-12-27T12:27:14.766861Z","iopub.status.idle":"2021-12-27T12:27:14.804653Z","shell.execute_reply.started":"2021-12-27T12:27:14.766819Z","shell.execute_reply":"2021-12-27T12:27:14.803948Z"},"trusted":true},"execution_count":38,"outputs":[{"execution_count":38,"output_type":"execute_result","data":{"text/plain":"   text_idx category                 aspect  target\n0      3976    Whole            [ресторане]       0\n1      3976    Whole           [ресторанах]       0\n2      3976    Whole            [ресторане]       0\n3      3976  Service  [столик, бронировали]       1\n4      3976  Service        [администратор]       1","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text_idx</th>\n      <th>category</th>\n      <th>aspect</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>[ресторане]</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>[ресторанах]</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3976</td>\n      <td>Whole</td>\n      <td>[ресторане]</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>[столик, бронировали]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3976</td>\n      <td>Service</td>\n      <td>[администратор]</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"#### Fasttext-эмбеддинги","metadata":{"id":"8JFzzcjPDvWX"}},{"cell_type":"code","source":"!wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ru.300.bin.gz","metadata":{"id":"YQ2f1SXHDzdS","outputId":"646d5b57-581f-4518-8f61-83844809cfa5","execution":{"iopub.status.busy":"2021-12-27T12:27:14.806137Z","iopub.execute_input":"2021-12-27T12:27:14.806587Z","iopub.status.idle":"2021-12-27T12:30:48.451502Z","shell.execute_reply.started":"2021-12-27T12:27:14.806552Z","shell.execute_reply":"2021-12-27T12:30:48.450689Z"},"trusted":true},"execution_count":39,"outputs":[{"name":"stdout","text":"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\nTo disable this warning, you can either:\n\t- Avoid using `tokenizers` before the fork if possible\n\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n--2021-12-27 12:27:15--  https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.ru.300.bin.gz\nResolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 104.22.75.142, 172.67.9.4, ...\nConnecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 4496459151 (4.2G) [application/octet-stream]\nSaving to: ‘cc.ru.300.bin.gz’\n\ncc.ru.300.bin.gz    100%[===================>]   4.19G  20.4MB/s    in 3m 32s  \n\n2021-12-27 12:30:48 (20.2 MB/s) - ‘cc.ru.300.bin.gz’ saved [4496459151/4496459151]\n\n","output_type":"stream"}]},{"cell_type":"code","source":"!gunzip cc.ru.300.bin.gz","metadata":{"id":"VsHUcWjZD27e","execution":{"iopub.status.busy":"2021-12-27T12:30:48.452900Z","iopub.execute_input":"2021-12-27T12:30:48.453123Z","iopub.status.idle":"2021-12-27T12:32:12.496655Z","shell.execute_reply.started":"2021-12-27T12:30:48.453096Z","shell.execute_reply":"2021-12-27T12:32:12.495820Z"},"trusted":true},"execution_count":40,"outputs":[{"name":"stdout","text":"huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\nTo disable this warning, you can either:\n\t- Avoid using `tokenizers` before the fork if possible\n\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n","output_type":"stream"}]},{"cell_type":"code","source":"ft_model = fasttext.load_model('cc.ru.300.bin')","metadata":{"id":"8DCrSUAsD3fu","outputId":"0f38e608-3e5a-4138-e2f1-62a1d1ce9b9f","execution":{"iopub.status.busy":"2021-12-27T12:32:12.500120Z","iopub.execute_input":"2021-12-27T12:32:12.500425Z","iopub.status.idle":"2021-12-27T12:33:54.414840Z","shell.execute_reply.started":"2021-12-27T12:32:12.500387Z","shell.execute_reply":"2021-12-27T12:33:54.414097Z"},"trusted":true},"execution_count":41,"outputs":[{"name":"stderr","text":"Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n","output_type":"stream"}]},{"cell_type":"code","source":"ft_model['ристоран'].shape","metadata":{"id":"eXRv_9e5D49_","outputId":"e504706f-b984-4546-a3b3-05a470fad68f","execution":{"iopub.status.busy":"2021-12-27T12:33:54.416004Z","iopub.execute_input":"2021-12-27T12:33:54.416688Z","iopub.status.idle":"2021-12-27T12:33:54.440481Z","shell.execute_reply.started":"2021-12-27T12:33:54.416648Z","shell.execute_reply":"2021-12-27T12:33:54.439704Z"},"trusted":true},"execution_count":42,"outputs":[{"execution_count":42,"output_type":"execute_result","data":{"text/plain":"(300,)"},"metadata":{}}]},{"cell_type":"markdown","source":"#### Обучение\nЧтобы обучить классификатор, поделим данные на обучающие и тестовые.","metadata":{"id":"NxLCht8kD7jw"}},{"cell_type":"code","source":"# тут мы поняли, что поделить данные мы поделили, а вот сохранили без индексов))\ntest_indices = [30808, 2495, 6668, 34956, 27629, 6376, 3152, 22015, 7824, 19503,\n                12203, 12341, 2692, 1751, 1511, 26887, 3906, 21059, 33252,\n                33816, 11770, 15335, 19746, 35904, 785, 1749, 17999, 19817,\n                36948, 34402, 3552, 11355, 33798, 2006, 2364, 10825, 37070,\n                35620, 1308, 16378, 7193, 7154, 33043]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:33:54.441960Z","iopub.execute_input":"2021-12-27T12:33:54.442483Z","iopub.status.idle":"2021-12-27T12:33:54.449711Z","shell.execute_reply.started":"2021-12-27T12:33:54.442431Z","shell.execute_reply":"2021-12-27T12:33:54.448836Z"},"trusted":true},"execution_count":43,"outputs":[]},{"cell_type":"code","source":"test_df = asp_df[asp_df['text_idx'].isin(test_indices)]\ntrain_df = asp_df[~asp_df['text_idx'].isin(test_indices)]\n\ntest_df","metadata":{"id":"fhcT1mIKCr1A","outputId":"c8b475d3-baf5-4a5c-ea82-74c43861a2a0","execution":{"iopub.status.busy":"2021-12-27T12:33:54.454730Z","iopub.execute_input":"2021-12-27T12:33:54.454930Z","iopub.status.idle":"2021-12-27T12:33:54.481006Z","shell.execute_reply.started":"2021-12-27T12:33:54.454906Z","shell.execute_reply":"2021-12-27T12:33:54.480245Z"},"trusted":true},"execution_count":44,"outputs":[{"execution_count":44,"output_type":"execute_result","data":{"text/plain":"      text_idx  category                       aspect  target\n20       30808     Whole                  [ресторане]       0\n21       30808  Interior              [первом, этаже]       3\n22       30808     Whole     [руководству, ресторана]       0\n23       30808   Service  [обслуживающему, персоналу]       1\n24       30808   Service                [сотрудникам]       1\n...        ...       ...                          ...     ...\n4751     33043   Service                      [заказ]       1\n4752     33043   Service                   [принесли]       1\n4753     33043      Food                [приготовили]       2\n4754     33043   Service                    [оставил]       1\n4755     33043     Whole                      [шатер]       0\n\n[677 rows x 4 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text_idx</th>\n      <th>category</th>\n      <th>aspect</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>20</th>\n      <td>30808</td>\n      <td>Whole</td>\n      <td>[ресторане]</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>30808</td>\n      <td>Interior</td>\n      <td>[первом, этаже]</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>30808</td>\n      <td>Whole</td>\n      <td>[руководству, ресторана]</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>30808</td>\n      <td>Service</td>\n      <td>[обслуживающему, персоналу]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>30808</td>\n      <td>Service</td>\n      <td>[сотрудникам]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4751</th>\n      <td>33043</td>\n      <td>Service</td>\n      <td>[заказ]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4752</th>\n      <td>33043</td>\n      <td>Service</td>\n      <td>[принесли]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4753</th>\n      <td>33043</td>\n      <td>Food</td>\n      <td>[приготовили]</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4754</th>\n      <td>33043</td>\n      <td>Service</td>\n      <td>[оставил]</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4755</th>\n      <td>33043</td>\n      <td>Whole</td>\n      <td>[шатер]</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>677 rows × 4 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"x_train = []\nfor wordlist in train_df['aspect']:\n    if len(wordlist) == 1:\n        vector = ft_model[wordlist[0]]\n        x_train.append(vector)\n    else:\n        vectors = [ft_model[word] for word in wordlist]\n        vector = np.mean(vectors, axis=0)\n        x_train.append(vector)\n\nx_test = []\nfor wordlist in test_df['aspect']:\n    if len(wordlist) == 1:\n        vector = ft_model[wordlist[0]]\n        x_test.append(vector)\n    else:\n        vectors = [ft_model[word] for word in wordlist]\n        vector = np.mean(vectors, axis=0)\n        x_test.append(vector)\n\ny_train = train_df['target'].values\ny_test = test_df['target'].values","metadata":{"id":"xGSgTV9PEOWy","execution":{"iopub.status.busy":"2021-12-27T12:33:54.482196Z","iopub.execute_input":"2021-12-27T12:33:54.482717Z","iopub.status.idle":"2021-12-27T12:33:54.596995Z","shell.execute_reply.started":"2021-12-27T12:33:54.482676Z","shell.execute_reply":"2021-12-27T12:33:54.596213Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"classifiers = [LogisticRegression(random_state=200),\n               MLPClassifier(max_iter=300, random_state=200),\n               GaussianNB(),\n               KNeighborsClassifier(), SVC(random_state=200),\n               DecisionTreeClassifier(random_state=200)]\nnames = []\nf1s = []\naccuracies = []\nfor clf in tqdm(classifiers):\n    clf.fit(x_train, y_train)\n    preds = clf.predict(x_test)\n    f1 = f1_score(y_test, preds, average='weighted')\n    accuracy = f1_score(y_test, preds, average='micro')\n\n    names.append(str(clf))\n    f1s.append(f1)\n    accuracies.append(accuracy)","metadata":{"id":"FDuY2AB3EcWB","outputId":"ae9ad694-9429-4769-e5b2-49d7e664eab0","execution":{"iopub.status.busy":"2021-12-27T12:33:54.598587Z","iopub.execute_input":"2021-12-27T12:33:54.598866Z","iopub.status.idle":"2021-12-27T12:34:13.257477Z","shell.execute_reply.started":"2021-12-27T12:33:54.598829Z","shell.execute_reply":"2021-12-27T12:34:13.256670Z"},"trusted":true},"execution_count":46,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae1754efa2a941fbbec01cbdc30ab493"}},"metadata":{}}]},{"cell_type":"code","source":"clf_df = pd.DataFrame({'classifier': names,\n                        'f1': f1s,\n                        'accuracy': accuracies})\nclf_df.style.highlight_max(['f1', 'accuracy'])","metadata":{"id":"MeKk6a5ZElpY","outputId":"e798802d-ba9c-4fcc-a146-574e3d83618d","execution":{"iopub.status.busy":"2021-12-27T12:34:13.258742Z","iopub.execute_input":"2021-12-27T12:34:13.259078Z","iopub.status.idle":"2021-12-27T12:34:13.327091Z","shell.execute_reply.started":"2021-12-27T12:34:13.259038Z","shell.execute_reply":"2021-12-27T12:34:13.326456Z"},"trusted":true},"execution_count":47,"outputs":[{"execution_count":47,"output_type":"execute_result","data":{"text/plain":"<pandas.io.formats.style.Styler at 0x7fdbd3a3c7d0>","text/html":"<style type=\"text/css\">\n#T_fcaa0_row4_col1, #T_fcaa0_row4_col2 {\n  background-color: yellow;\n}\n</style>\n<table id=\"T_fcaa0_\">\n  <thead>\n    <tr>\n      <th class=\"blank level0\" >&nbsp;</th>\n      <th class=\"col_heading level0 col0\" >classifier</th>\n      <th class=\"col_heading level0 col1\" >f1</th>\n      <th class=\"col_heading level0 col2\" >accuracy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th id=\"T_fcaa0_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n      <td id=\"T_fcaa0_row0_col0\" class=\"data row0 col0\" >LogisticRegression(random_state=200)</td>\n      <td id=\"T_fcaa0_row0_col1\" class=\"data row0 col1\" >0.925796</td>\n      <td id=\"T_fcaa0_row0_col2\" class=\"data row0 col2\" >0.926145</td>\n    </tr>\n    <tr>\n      <th id=\"T_fcaa0_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n      <td id=\"T_fcaa0_row1_col0\" class=\"data row1 col0\" >MLPClassifier(max_iter=300, random_state=200)</td>\n      <td id=\"T_fcaa0_row1_col1\" class=\"data row1 col1\" >0.934825</td>\n      <td id=\"T_fcaa0_row1_col2\" class=\"data row1 col2\" >0.935007</td>\n    </tr>\n    <tr>\n      <th id=\"T_fcaa0_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n      <td id=\"T_fcaa0_row2_col0\" class=\"data row2 col0\" >GaussianNB()</td>\n      <td id=\"T_fcaa0_row2_col1\" class=\"data row2 col1\" >0.801363</td>\n      <td id=\"T_fcaa0_row2_col2\" class=\"data row2 col2\" >0.800591</td>\n    </tr>\n    <tr>\n      <th id=\"T_fcaa0_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n      <td id=\"T_fcaa0_row3_col0\" class=\"data row3 col0\" >KNeighborsClassifier()</td>\n      <td id=\"T_fcaa0_row3_col1\" class=\"data row3 col1\" >0.910671</td>\n      <td id=\"T_fcaa0_row3_col2\" class=\"data row3 col2\" >0.911374</td>\n    </tr>\n    <tr>\n      <th id=\"T_fcaa0_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n      <td id=\"T_fcaa0_row4_col0\" class=\"data row4 col0\" >SVC(random_state=200)</td>\n      <td id=\"T_fcaa0_row4_col1\" class=\"data row4 col1\" >0.940610</td>\n      <td id=\"T_fcaa0_row4_col2\" class=\"data row4 col2\" >0.940916</td>\n    </tr>\n    <tr>\n      <th id=\"T_fcaa0_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n      <td id=\"T_fcaa0_row5_col0\" class=\"data row5 col0\" >DecisionTreeClassifier(random_state=200)</td>\n      <td id=\"T_fcaa0_row5_col1\" class=\"data row5 col1\" >0.889412</td>\n      <td id=\"T_fcaa0_row5_col2\" class=\"data row5 col2\" >0.889217</td>\n    </tr>\n  </tbody>\n</table>\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Видим, что для нашей задачи хороши MLP и SVC (на другой выборке MLP лидировал с незначительными отличиями).","metadata":{"id":"ARIfuDbTG58D"}},{"cell_type":"markdown","source":"#### Ошибки SVC","metadata":{"id":"ko_OQIpdExQ2"}},{"cell_type":"code","source":"svc = SVC(random_state=200)\nsvc.fit(x_train, y_train)\n\npreds = svc.predict(x_test)\nprint(f1_score(y_test, preds, average='weighted'))\nprint(f1_score(y_test, preds, average='micro'))","metadata":{"id":"HAqNakE1Esl_","outputId":"185d2cf0-5787-4a14-8d86-0d164d7cc34d","execution":{"iopub.status.busy":"2021-12-27T12:34:13.328165Z","iopub.execute_input":"2021-12-27T12:34:13.328417Z","iopub.status.idle":"2021-12-27T12:34:15.737519Z","shell.execute_reply.started":"2021-12-27T12:34:13.328383Z","shell.execute_reply":"2021-12-27T12:34:15.736767Z"},"trusted":true},"execution_count":48,"outputs":[{"name":"stdout","text":"0.9406100129182599\n0.9409158050221565\n","output_type":"stream"}]},{"cell_type":"code","source":"inv_mapper = {i: target for target, i in list(mapper.items())}","metadata":{"id":"dpxRtXFCIA0T","execution":{"iopub.status.busy":"2021-12-27T12:34:15.738881Z","iopub.execute_input":"2021-12-27T12:34:15.739339Z","iopub.status.idle":"2021-12-27T12:34:15.743947Z","shell.execute_reply.started":"2021-12-27T12:34:15.739289Z","shell.execute_reply":"2021-12-27T12:34:15.743198Z"},"trusted":true},"execution_count":49,"outputs":[]},{"cell_type":"code","source":"mistakes = []\ntest_aspects = test_df['aspect'].tolist()\nfor i, (true, pred) in enumerate(zip(y_test, preds)):\n    if true != pred:\n        mistakes.append((inv_mapper[true], inv_mapper[pred], test_aspects[i]))","metadata":{"id":"9U76UX3UHQeJ","execution":{"iopub.status.busy":"2021-12-27T12:34:15.745282Z","iopub.execute_input":"2021-12-27T12:34:15.745932Z","iopub.status.idle":"2021-12-27T12:34:15.754041Z","shell.execute_reply.started":"2021-12-27T12:34:15.745877Z","shell.execute_reply":"2021-12-27T12:34:15.753281Z"},"trusted":true},"execution_count":50,"outputs":[]},{"cell_type":"code","source":"mistakes_df = pd.DataFrame(mistakes, columns=['true', 'pred', 'aspect'])\nmistakes_df","metadata":{"id":"wWyV3jTNHq2C","outputId":"5c934bb7-5f08-44a4-b0dc-a6a7ca94e301","execution":{"iopub.status.busy":"2021-12-27T12:34:15.755131Z","iopub.execute_input":"2021-12-27T12:34:15.755643Z","iopub.status.idle":"2021-12-27T12:34:15.784509Z","shell.execute_reply.started":"2021-12-27T12:34:15.755606Z","shell.execute_reply":"2021-12-27T12:34:15.782721Z"},"trusted":true},"execution_count":51,"outputs":[{"execution_count":51,"output_type":"execute_result","data":{"text/plain":"        true      pred                                   aspect\n0      Whole   Service                 [руководству, ресторана]\n1       Food  Interior                                   [стол]\n2    Service      Food                                [поварам]\n3       Food   Service                              [бивштексы]\n4      Whole      Food                           [квартиру, 55]\n5      Whole      Food    [сетевых, заведений, японской, кухни]\n6   Interior   Service                      [караоке-программа]\n7      Price     Whole                    [оставили, 10, тысяч]\n8      Whole  Interior                      [чердак, художника]\n9   Interior     Whole                              [обтановка]\n10   Service      Food                                 [повара]\n11     Whole      Food           [ресторанов, китайской, кухни]\n12   Service     Whole                               [ресторан]\n13  Interior     Whole                                  [места]\n14      Food   Service             [пирожок, к, первому, блюду]\n15  Interior   Service                              [программа]\n16  Interior   Service    [грузинские, танцевальный, коллектив]\n17      Food     Whole                              [пивоварни]\n18   Service      Food          [блюда, и, напитки, выносились]\n19  Interior   Service  [развлекательная, программа, ресторана]\n20   Service     Whole                     [персоналу, паберти]\n21  Interior   Service                         [девушка-певица]\n22  Interior   Service                            [потанцевали]\n23      Food     Whole                                  [шурвы]\n24      Food   Service           [выбор, по, меню, и, напиткам]\n25  Interior     Whole                                  [места]\n26     Whole  Interior                     [патио, на, невском]\n27     Whole      Food                          [планета, суши]\n28   Service     Price                                   [счёт]\n29     Price   Service                                  [сдачу]\n30     Price   Service                       [скидочную, карту]\n31  Interior   Service                              [программу]\n32     Whole   Service                              [официанты]\n33      Food     Whole                                [америка]\n34      Food     Whole                      [ассортимент, бара]\n35   Service      Food                   [подготовили, закуски]\n36   Service      Food                                 [подача]\n37  Interior     Whole                                [колонку]\n38  Interior     Whole                                  [место]\n39     Whole  Interior                                  [шатер]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>true</th>\n      <th>pred</th>\n      <th>aspect</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Whole</td>\n      <td>Service</td>\n      <td>[руководству, ресторана]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Food</td>\n      <td>Interior</td>\n      <td>[стол]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Service</td>\n      <td>Food</td>\n      <td>[поварам]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Food</td>\n      <td>Service</td>\n      <td>[бивштексы]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Whole</td>\n      <td>Food</td>\n      <td>[квартиру, 55]</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Whole</td>\n      <td>Food</td>\n      <td>[сетевых, заведений, японской, кухни]</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[караоке-программа]</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Price</td>\n      <td>Whole</td>\n      <td>[оставили, 10, тысяч]</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Whole</td>\n      <td>Interior</td>\n      <td>[чердак, художника]</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Interior</td>\n      <td>Whole</td>\n      <td>[обтановка]</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Service</td>\n      <td>Food</td>\n      <td>[повара]</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>Whole</td>\n      <td>Food</td>\n      <td>[ресторанов, китайской, кухни]</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>Service</td>\n      <td>Whole</td>\n      <td>[ресторан]</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>Interior</td>\n      <td>Whole</td>\n      <td>[места]</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>Food</td>\n      <td>Service</td>\n      <td>[пирожок, к, первому, блюду]</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[программа]</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[грузинские, танцевальный, коллектив]</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>Food</td>\n      <td>Whole</td>\n      <td>[пивоварни]</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>Service</td>\n      <td>Food</td>\n      <td>[блюда, и, напитки, выносились]</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[развлекательная, программа, ресторана]</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>Service</td>\n      <td>Whole</td>\n      <td>[персоналу, паберти]</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[девушка-певица]</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[потанцевали]</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>Food</td>\n      <td>Whole</td>\n      <td>[шурвы]</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>Food</td>\n      <td>Service</td>\n      <td>[выбор, по, меню, и, напиткам]</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>Interior</td>\n      <td>Whole</td>\n      <td>[места]</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>Whole</td>\n      <td>Interior</td>\n      <td>[патио, на, невском]</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>Whole</td>\n      <td>Food</td>\n      <td>[планета, суши]</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>Service</td>\n      <td>Price</td>\n      <td>[счёт]</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>Price</td>\n      <td>Service</td>\n      <td>[сдачу]</td>\n    </tr>\n    <tr>\n      <th>30</th>\n      <td>Price</td>\n      <td>Service</td>\n      <td>[скидочную, карту]</td>\n    </tr>\n    <tr>\n      <th>31</th>\n      <td>Interior</td>\n      <td>Service</td>\n      <td>[программу]</td>\n    </tr>\n    <tr>\n      <th>32</th>\n      <td>Whole</td>\n      <td>Service</td>\n      <td>[официанты]</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>Food</td>\n      <td>Whole</td>\n      <td>[америка]</td>\n    </tr>\n    <tr>\n      <th>34</th>\n      <td>Food</td>\n      <td>Whole</td>\n      <td>[ассортимент, бара]</td>\n    </tr>\n    <tr>\n      <th>35</th>\n      <td>Service</td>\n      <td>Food</td>\n      <td>[подготовили, закуски]</td>\n    </tr>\n    <tr>\n      <th>36</th>\n      <td>Service</td>\n      <td>Food</td>\n      <td>[подача]</td>\n    </tr>\n    <tr>\n      <th>37</th>\n      <td>Interior</td>\n      <td>Whole</td>\n      <td>[колонку]</td>\n    </tr>\n    <tr>\n      <th>38</th>\n      <td>Interior</td>\n      <td>Whole</td>\n      <td>[место]</td>\n    </tr>\n    <tr>\n      <th>39</th>\n      <td>Whole</td>\n      <td>Interior</td>\n      <td>[шатер]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Классифицируем наши аспекты\nУра, у нас есть классификатор! Применим его к полученным ранее аспектам.","metadata":{"id":"lgDHvb1oMjTN"}},{"cell_type":"code","source":"def prepare_for_classification(aspects_list):\n    aspects_sent = [asp['asp_text'] for asp in aspects_list]\n    asp_test = []\n    # если в предложении нет аспектов, возвращаем вектор нулей\n    # потом просто выбросим это предсказание\n    if not aspects_sent:\n        asp_test = [np.zeros(300)]\n    for aspect in aspects_sent:\n        asp_parts = aspect.split()\n        if len(asp_parts) == 1:\n            vector = ft_model[asp_parts[0]]\n            asp_test.append(vector)\n        else:\n            vectors = [ft_model[word] for word in asp_parts]\n            vector = np.mean(vectors, axis=0)\n            asp_test.append(vector)\n    return asp_test","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:15.785902Z","iopub.execute_input":"2021-12-27T12:34:15.786690Z","iopub.status.idle":"2021-12-27T12:34:15.795154Z","shell.execute_reply.started":"2021-12-27T12:34:15.786653Z","shell.execute_reply":"2021-12-27T12:34:15.794277Z"},"trusted":true},"execution_count":52,"outputs":[]},{"cell_type":"code","source":"full_aspects_for_clf = []\nfor one_text_asps in text_aspects:\n    asps_for_clf = []\n    for one_sent_asps in one_text_asps:\n        asps_for_clf.append(prepare_for_classification(one_sent_asps))\n    full_aspects_for_clf.append(asps_for_clf)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:15.796442Z","iopub.execute_input":"2021-12-27T12:34:15.797144Z","iopub.status.idle":"2021-12-27T12:34:15.835428Z","shell.execute_reply.started":"2021-12-27T12:34:15.797107Z","shell.execute_reply":"2021-12-27T12:34:15.834752Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"def classify_aspects(aspects_for_clf, svc, inv_mapper):\n    preds_by_sent = []\n    for sent_asps in aspects_for_clf:\n        preds = svc.predict(sent_asps)\n        mapped_preds = [inv_mapper[pred] for pred in preds]\n        preds_by_sent.append(mapped_preds)\n    return preds_by_sent","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:15.836519Z","iopub.execute_input":"2021-12-27T12:34:15.836733Z","iopub.status.idle":"2021-12-27T12:34:15.842391Z","shell.execute_reply.started":"2021-12-27T12:34:15.836702Z","shell.execute_reply":"2021-12-27T12:34:15.841384Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"code","source":"all_pred_labels = []\nfor one_text in full_aspects_for_clf:\n    pred_labels = classify_aspects(one_text, svc, inv_mapper)\n    all_pred_labels.append(pred_labels)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:15.843786Z","iopub.execute_input":"2021-12-27T12:34:15.843972Z","iopub.status.idle":"2021-12-27T12:34:16.590461Z","shell.execute_reply.started":"2021-12-27T12:34:15.843942Z","shell.execute_reply":"2021-12-27T12:34:16.589761Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"all_pred_labels[0]  # для первого текста","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.591801Z","iopub.execute_input":"2021-12-27T12:34:16.592159Z","iopub.status.idle":"2021-12-27T12:34:16.598690Z","shell.execute_reply.started":"2021-12-27T12:34:16.592117Z","shell.execute_reply":"2021-12-27T12:34:16.597832Z"},"trusted":true},"execution_count":56,"outputs":[{"execution_count":56,"output_type":"execute_result","data":{"text/plain":"[['Whole'],\n ['Service'],\n ['Service', 'Service', 'Service', 'Service', 'Service'],\n ['Service'],\n ['Service', 'Service'],\n ['Service'],\n ['Whole', 'Whole'],\n ['Service'],\n ['Food', 'Price'],\n ['Whole', 'Food', 'Price', 'Food', 'Service'],\n ['Whole']]"},"metadata":{}}]},{"cell_type":"markdown","source":"Добавляем предсказанные классы к выделенным ранее аспектам:","metadata":{}},{"cell_type":"code","source":"for i, text in enumerate(text_aspects):\n    for j, sentence in enumerate(text):\n        for n, aspect in enumerate(sentence):\n            aspect['category'] = all_pred_labels[i][j][n]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.600052Z","iopub.execute_input":"2021-12-27T12:34:16.600322Z","iopub.status.idle":"2021-12-27T12:34:16.609060Z","shell.execute_reply.started":"2021-12-27T12:34:16.600288Z","shell.execute_reply":"2021-12-27T12:34:16.608400Z"},"trusted":true},"execution_count":57,"outputs":[]},{"cell_type":"code","source":"text_aspects[0][0:3]  # аспекты первых 3 предложений первого текста","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.610444Z","iopub.execute_input":"2021-12-27T12:34:16.610754Z","iopub.status.idle":"2021-12-27T12:34:16.620432Z","shell.execute_reply.started":"2021-12-27T12:34:16.610716Z","shell.execute_reply":"2021-12-27T12:34:16.619580Z"},"trusted":true},"execution_count":58,"outputs":[{"execution_count":58,"output_type":"execute_result","data":{"text/plain":"[[{'asp_idx': 13823,\n   'asp_text': 'аппетит \"',\n   'asp_start': 8,\n   'asp_end': 16,\n   'sentiment1': 'neutral',\n   'category': 'Whole'}],\n [],\n [{'asp_idx': 13823,\n   'asp_text': 'встретил',\n   'asp_start': 138,\n   'asp_end': 146,\n   'sentiment1': 'neutral',\n   'category': 'Service'},\n  {'asp_idx': 13823,\n   'asp_text': 'менеджер',\n   'asp_start': 147,\n   'asp_end': 155,\n   'sentiment1': 'neutral',\n   'category': 'Service'},\n  {'asp_idx': 13823,\n   'asp_text': 'девушка',\n   'asp_start': 179,\n   'asp_end': 186,\n   'sentiment1': 'neutral',\n   'category': 'Service'},\n  {'asp_idx': 13823,\n   'asp_text': 'проводила к столу',\n   'asp_start': 188,\n   'asp_end': 205,\n   'sentiment1': 'neutral',\n   'category': 'Service'},\n  {'asp_idx': 13823,\n   'asp_text': 'дала меню',\n   'asp_start': 208,\n   'asp_end': 217,\n   'sentiment1': 'neutral',\n   'category': 'Service'}]]"},"metadata":{}}]},{"cell_type":"markdown","source":"Для подсчета тональности по категориям пока выбран самый простой способ: нет упоминания -- *absence*, все упоминания одной тональности -- эта тональность, есть упоминания разных тональностей -- *both*.","metadata":{}},{"cell_type":"code","source":"text_sentiments = {}\n\nfor text in text_aspects:\n    text_sent = {}\n    whole, interior, food, service, price = [], [], [], [], []\n    \n    for sentence in text:\n\n        for aspect in sentence:\n\n            text_idx = aspect['asp_idx']\n            categ = aspect['category']\n            sentim = aspect['sentiment1']\n            if categ == 'Whole':\n                whole.append(sentim)\n            elif categ == 'Interior':\n                interior.append(sentim)\n            elif categ == 'Food':\n                food.append(sentim)\n            elif categ == 'Service':\n                service.append(sentim)\n            elif categ == 'Price':\n                price.append(sentim)\n\n    food = Counter(food).most_common()\n    interior = Counter(interior).most_common()\n    price = Counter(price).most_common()\n    whole = Counter(whole).most_common()\n    service = Counter(service).most_common()\n\n    if len(food) == 0:\n        text_sent['Food'] = 'absence'\n    elif len(food) == 1:\n        text_sent['Food'] = food[0][0]\n    elif food[0][1] > food[1][1]:\n        text_sent['Food'] = food[0][0]\n    else:\n        text_sent['Food'] = 'both'\n\n    if len(interior) == 0:\n        text_sent['Interior'] = 'absence'\n    elif len(interior) == 1:\n        text_sent['Interior'] = interior[0][0]\n    elif interior[0][1] > interior[1][1]:\n        text_sent['Interior'] = interior[0][0]\n    else:\n        text_sent['Interior'] = 'both'\n\n    if len(price) == 0:\n        text_sent['Price'] = 'absence'\n    elif len(price) == 1:\n        text_sent['Price'] = price[0][0]\n    elif price[0][1] > price[1][1]:\n        text_sent['Price'] = price[0][0]\n    else:\n        text_sent['Price'] = 'both'\n\n    if len(whole) == 0:\n        text_sent['Whole'] = 'absence'\n    elif len(whole) == 1:\n        text_sent['Whole'] = whole[0][0]\n    elif whole[0][1] > whole[1][1]:\n        text_sent['Whole'] = whole[0][0]\n    else:\n        text_sent['Whole'] = 'both'\n\n    if len(service) == 0:\n        text_sent['Service'] = 'absence'\n    elif len(service) == 1:\n        text_sent['Service'] = service[0][0]\n    elif service[0][1] > service[1][1]:\n        text_sent['Service'] = service[0][0]\n    else:\n        text_sent['Service'] = 'both'\n\n    text_sentiments[text_idx] = text_sent","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.622812Z","iopub.execute_input":"2021-12-27T12:34:16.623037Z","iopub.status.idle":"2021-12-27T12:34:16.643571Z","shell.execute_reply.started":"2021-12-27T12:34:16.623013Z","shell.execute_reply":"2021-12-27T12:34:16.641544Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"text_sentiments[13823]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.645117Z","iopub.execute_input":"2021-12-27T12:34:16.645551Z","iopub.status.idle":"2021-12-27T12:34:16.655599Z","shell.execute_reply.started":"2021-12-27T12:34:16.645515Z","shell.execute_reply":"2021-12-27T12:34:16.654871Z"},"trusted":true},"execution_count":60,"outputs":[{"execution_count":60,"output_type":"execute_result","data":{"text/plain":"{'Food': 'positive',\n 'Interior': 'absence',\n 'Price': 'positive',\n 'Whole': 'positive',\n 'Service': 'neutral'}"},"metadata":{}}]},{"cell_type":"markdown","source":"Запишем результаты разметки в итоговые файлы.","metadata":{}},{"cell_type":"code","source":"all_aspects_with_cats = []\nfor text in text_aspects:\n    for sent in text:\n        all_aspects_with_cats.extend(sent)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.656943Z","iopub.execute_input":"2021-12-27T12:34:16.657456Z","iopub.status.idle":"2021-12-27T12:34:16.663934Z","shell.execute_reply.started":"2021-12-27T12:34:16.657421Z","shell.execute_reply":"2021-12-27T12:34:16.663285Z"},"trusted":true},"execution_count":61,"outputs":[]},{"cell_type":"code","source":"all_aspects_with_cats[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.664791Z","iopub.execute_input":"2021-12-27T12:34:16.666402Z","iopub.status.idle":"2021-12-27T12:34:16.674661Z","shell.execute_reply.started":"2021-12-27T12:34:16.666364Z","shell.execute_reply":"2021-12-27T12:34:16.673940Z"},"trusted":true},"execution_count":62,"outputs":[{"execution_count":62,"output_type":"execute_result","data":{"text/plain":"{'asp_idx': 13823,\n 'asp_text': 'аппетит \"',\n 'asp_start': 8,\n 'asp_end': 16,\n 'sentiment1': 'neutral',\n 'category': 'Whole'}"},"metadata":{}}]},{"cell_type":"code","source":"full_tagged_df = pd.DataFrame(all_aspects_with_cats)\nfull_tagged_df = full_tagged_df[['asp_idx', 'category', 'asp_text',\n                                 'asp_start', 'asp_end', 'sentiment1']]\nfull_tagged_df","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.676399Z","iopub.execute_input":"2021-12-27T12:34:16.677569Z","iopub.status.idle":"2021-12-27T12:34:16.700068Z","shell.execute_reply.started":"2021-12-27T12:34:16.677458Z","shell.execute_reply":"2021-12-27T12:34:16.699257Z"},"trusted":true},"execution_count":63,"outputs":[{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"      asp_idx category                                  asp_text  asp_start  \\\n0       13823    Whole                                 аппетит \"          8   \n1       13823  Service                                  встретил        138   \n2       13823  Service                                  менеджер        147   \n3       13823  Service                                   девушка        179   \n4       13823  Service                         проводила к столу        188   \n...       ...      ...                                       ...        ...   \n1175    11770     Food                                     стейк        831   \n1176    11770     Food  блюдо тартар с сырой рыбой и сырым яйцом        896   \n1177    11770  Service                                 Официанты        938   \n1178    11770  Service                               обстановкая        976   \n1179    11770    Whole                                  ресторан       1007   \n\n      asp_end sentiment1  \n0          16    neutral  \n1         146    neutral  \n2         155    neutral  \n3         186    neutral  \n4         205    neutral  \n...       ...        ...  \n1175      836   positive  \n1176      936   positive  \n1177      947   positive  \n1178      987   positive  \n1179     1015   positive  \n\n[1180 rows x 6 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>asp_idx</th>\n      <th>category</th>\n      <th>asp_text</th>\n      <th>asp_start</th>\n      <th>asp_end</th>\n      <th>sentiment1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>13823</td>\n      <td>Whole</td>\n      <td>аппетит \"</td>\n      <td>8</td>\n      <td>16</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>13823</td>\n      <td>Service</td>\n      <td>встретил</td>\n      <td>138</td>\n      <td>146</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>13823</td>\n      <td>Service</td>\n      <td>менеджер</td>\n      <td>147</td>\n      <td>155</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>13823</td>\n      <td>Service</td>\n      <td>девушка</td>\n      <td>179</td>\n      <td>186</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>13823</td>\n      <td>Service</td>\n      <td>проводила к столу</td>\n      <td>188</td>\n      <td>205</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1175</th>\n      <td>11770</td>\n      <td>Food</td>\n      <td>стейк</td>\n      <td>831</td>\n      <td>836</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>1176</th>\n      <td>11770</td>\n      <td>Food</td>\n      <td>блюдо тартар с сырой рыбой и сырым яйцом</td>\n      <td>896</td>\n      <td>936</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>1177</th>\n      <td>11770</td>\n      <td>Service</td>\n      <td>Официанты</td>\n      <td>938</td>\n      <td>947</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>1178</th>\n      <td>11770</td>\n      <td>Service</td>\n      <td>обстановкая</td>\n      <td>976</td>\n      <td>987</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>1179</th>\n      <td>11770</td>\n      <td>Whole</td>\n      <td>ресторан</td>\n      <td>1007</td>\n      <td>1015</td>\n      <td>positive</td>\n    </tr>\n  </tbody>\n</table>\n<p>1180 rows × 6 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"full_tagged_df.to_csv('sentiment_by_aspect.tsv', sep='\\t', index=False, header=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.701787Z","iopub.execute_input":"2021-12-27T12:34:16.702224Z","iopub.status.idle":"2021-12-27T12:34:16.716130Z","shell.execute_reply.started":"2021-12-27T12:34:16.702172Z","shell.execute_reply":"2021-12-27T12:34:16.715534Z"},"trusted":true},"execution_count":64,"outputs":[]},{"cell_type":"code","source":"categs_for_df = []\nfor text, sentiments in list(text_sentiments.items()):\n    for categ, sentim in list(sentiments.items()):\n        categs_for_df.append([text, categ, sentim])","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.717306Z","iopub.execute_input":"2021-12-27T12:34:16.717716Z","iopub.status.idle":"2021-12-27T12:34:16.723003Z","shell.execute_reply.started":"2021-12-27T12:34:16.717682Z","shell.execute_reply":"2021-12-27T12:34:16.722222Z"},"trusted":true},"execution_count":65,"outputs":[]},{"cell_type":"code","source":"categ_tagged_df = pd.DataFrame(categs_for_df, columns=['idx', 'category', 'sentiment'])\ncateg_tagged_df","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.725067Z","iopub.execute_input":"2021-12-27T12:34:16.725661Z","iopub.status.idle":"2021-12-27T12:34:16.740982Z","shell.execute_reply.started":"2021-12-27T12:34:16.725630Z","shell.execute_reply":"2021-12-27T12:34:16.740189Z"},"trusted":true},"execution_count":66,"outputs":[{"execution_count":66,"output_type":"execute_result","data":{"text/plain":"       idx  category sentiment\n0    13823      Food  positive\n1    13823  Interior   absence\n2    13823     Price  positive\n3    13823     Whole  positive\n4    13823   Service   neutral\n..     ...       ...       ...\n350  11770      Food      both\n351  11770  Interior   absence\n352  11770     Price   absence\n353  11770     Whole   neutral\n354  11770   Service  positive\n\n[355 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>idx</th>\n      <th>category</th>\n      <th>sentiment</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>13823</td>\n      <td>Food</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>13823</td>\n      <td>Interior</td>\n      <td>absence</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>13823</td>\n      <td>Price</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>13823</td>\n      <td>Whole</td>\n      <td>positive</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>13823</td>\n      <td>Service</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>350</th>\n      <td>11770</td>\n      <td>Food</td>\n      <td>both</td>\n    </tr>\n    <tr>\n      <th>351</th>\n      <td>11770</td>\n      <td>Interior</td>\n      <td>absence</td>\n    </tr>\n    <tr>\n      <th>352</th>\n      <td>11770</td>\n      <td>Price</td>\n      <td>absence</td>\n    </tr>\n    <tr>\n      <th>353</th>\n      <td>11770</td>\n      <td>Whole</td>\n      <td>neutral</td>\n    </tr>\n    <tr>\n      <th>354</th>\n      <td>11770</td>\n      <td>Service</td>\n      <td>positive</td>\n    </tr>\n  </tbody>\n</table>\n<p>355 rows × 3 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"categ_tagged_df.to_csv('sentiment_by_category.tsv', sep='\\t', index=False, header=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.742566Z","iopub.execute_input":"2021-12-27T12:34:16.742982Z","iopub.status.idle":"2021-12-27T12:34:16.750957Z","shell.execute_reply.started":"2021-12-27T12:34:16.742945Z","shell.execute_reply":"2021-12-27T12:34:16.750141Z"},"trusted":true},"execution_count":67,"outputs":[]},{"cell_type":"markdown","source":"## Этап 5 (эксперимент): оцениваем тональность готовой моделью","metadata":{}},{"cell_type":"markdown","source":"Мы будем определять тональность не каждого слова, а n-граммы \"аспект + контекст\".\n\nЧто у нас есть?\n* test_with_sentiment -- потокенная разметка с делением на предложения и тексты\n* text_aspects -- аспекты в виде списков токенов с делением на предложения и тексты\n\nЧто нужно получить?\n* для каждого аспекта -- его контекст в рамках предложения\n* для пары \"аспект - контекст\" -- тональность с помощью [готовой модели](https://huggingface.co/blanchefort/rubert-base-cased-sentiment)\n* для каждого текста -- список входящих в него аспектов, их индексов и тональностей\n\nПроблема:\n* выбранная модель (и все другие, которые мы нашли) не умеет выделять *both* (такие употребления она помечает как *neutral*)\n* попробуем решить это, добавив свой способ выбора сентимента из полученных вероятностей","metadata":{}},{"cell_type":"code","source":"window = 3\nnew_text_aspects = []\n# идем по текстам\nfor text_idx, (rev_idx, text) in enumerate(list(test_with_sentiment.items())):\n    one_text_aspects = []  # аспекты одного предложения в контексте\n    # идем по предложению\n    for sent_idx, sentence in enumerate(list(text.values())):\n        one_sent_aspects = []\n        # получаем аспекты, относящиеся к этому предложению\n        sent_aspects = get_sentence_aspects(sentence)\n        # проходим по аспектам и получаем контекст\n        for aspect in sent_aspects:\n            asp_with_context = []  # сюда положим аспект с контекстом\n            first_word = aspect[0]['id']  # первое слово аспекта\n            last_word = aspect[-1]['id']  # последнее слово аспекта\n            first_context = first_word - window  # индекс первого слова контекста\n            # если мы в рамках предложения, берем нужные токены\n            for i in range(3):  # берем три слова до аспекта\n                if first_context >= 0:\n                    asp_with_context.append(sentence[first_context]['text'])\n                    first_context += 1  # переходим к следующему слову\n            for word in aspect:  # берем все слова аспекта\n                asp_with_context.append(word['text'])\n            last_context = last_word\n            for i in range(3):  # берем три слова после аспекта\n                last_context += 1\n                # проверяем, что предложение еще не кончилось\n                if last_context in set(sentence.keys()):\n                    asp_with_context.append(sentence[last_context]['text'])\n            asp_info = {'asp_idx': rev_idx,  # индекс ревью в файле\n                        'asp_text': ' '.join([pt['text'] for pt in aspect]),\n                        'asp_start': aspect[0]['start'],\n                        'asp_end': aspect[-1]['end'],\n                        'asp_with_context': ' '.join(asp_with_context)}\n            one_sent_aspects.append(asp_info)\n        one_text_aspects.append(one_sent_aspects)\n    new_text_aspects.append(one_text_aspects)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.752597Z","iopub.execute_input":"2021-12-27T12:34:16.752918Z","iopub.status.idle":"2021-12-27T12:34:16.784622Z","shell.execute_reply.started":"2021-12-27T12:34:16.752881Z","shell.execute_reply":"2021-12-27T12:34:16.783994Z"},"trusted":true},"execution_count":68,"outputs":[]},{"cell_type":"code","source":"new_text_aspects[0][2]  # аспекты третьего предложения первого текста","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.785975Z","iopub.execute_input":"2021-12-27T12:34:16.786242Z","iopub.status.idle":"2021-12-27T12:34:16.798397Z","shell.execute_reply.started":"2021-12-27T12:34:16.786196Z","shell.execute_reply":"2021-12-27T12:34:16.797268Z"},"trusted":true},"execution_count":69,"outputs":[{"execution_count":69,"output_type":"execute_result","data":{"text/plain":"[{'asp_idx': 13823,\n  'asp_text': 'встретил',\n  'asp_start': 138,\n  'asp_end': 146,\n  'asp_with_context': 'встретил менеджер - темноволосая'},\n {'asp_idx': 13823,\n  'asp_text': 'менеджер',\n  'asp_start': 147,\n  'asp_end': 155,\n  'asp_with_context': 'менеджер - темноволосая стройная'},\n {'asp_idx': 13823,\n  'asp_text': 'девушка',\n  'asp_start': 179,\n  'asp_end': 186,\n  'asp_with_context': '- темноволосая стройная девушка , проводила к'},\n {'asp_idx': 13823,\n  'asp_text': 'проводила к столу',\n  'asp_start': 188,\n  'asp_end': 205,\n  'asp_with_context': 'стройная девушка , проводила к столу и дала меню'},\n {'asp_idx': 13823,\n  'asp_text': 'дала меню',\n  'asp_start': 208,\n  'asp_end': 217,\n  'asp_with_context': 'к столу и дала меню . .'}]"},"metadata":{}}]},{"cell_type":"markdown","source":"Теперь получим для каждого аспекта в контексте его тональность.","metadata":{}},{"cell_type":"code","source":"tokenizer = BertTokenizerFast.from_pretrained(\n    'blanchefort/rubert-base-cased-sentiment')\npretrained_model = AutoModelForSequenceClassification.from_pretrained(\n    'blanchefort/rubert-base-cased-sentiment', return_dict=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:16.799865Z","iopub.execute_input":"2021-12-27T12:34:16.800125Z","iopub.status.idle":"2021-12-27T12:34:45.092908Z","shell.execute_reply.started":"2021-12-27T12:34:16.800090Z","shell.execute_reply":"2021-12-27T12:34:45.092204Z"},"trusted":true},"execution_count":70,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/1.34M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"da070d8d1c944feba0e7c26461e45e62"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/112 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5d0f4d320a5e499ca08ae3650771670e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/499 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b267720c37044f3d8e85f4133df07f91"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/943 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b8f325eaef2546f6bc0a18f81b087734"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/679M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7b21c0f46ad04db7b2be468eb8da639e"}},"metadata":{}}]},{"cell_type":"code","source":"@torch.no_grad()\ndef predict_sentiment(text, tokenizer, model):\n    inputs = tokenizer(text, max_length=512, padding=True,\n                       truncation=True, return_tensors='pt')\n    outputs = model(**inputs)\n    predicted = torch.nn.functional.softmax(outputs.logits, dim=1)\n    return predicted","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:45.097692Z","iopub.execute_input":"2021-12-27T12:34:45.097905Z","iopub.status.idle":"2021-12-27T12:34:45.102591Z","shell.execute_reply.started":"2021-12-27T12:34:45.097871Z","shell.execute_reply":"2021-12-27T12:34:45.101725Z"},"trusted":true},"execution_count":71,"outputs":[]},{"cell_type":"code","source":"predict_sentiment(new_text_aspects[0][0][0]['asp_with_context'],\n                  tokenizer, pretrained_model)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:45.103859Z","iopub.execute_input":"2021-12-27T12:34:45.104363Z","iopub.status.idle":"2021-12-27T12:34:45.279548Z","shell.execute_reply.started":"2021-12-27T12:34:45.104317Z","shell.execute_reply":"2021-12-27T12:34:45.278705Z"},"trusted":true},"execution_count":72,"outputs":[{"execution_count":72,"output_type":"execute_result","data":{"text/plain":"tensor([[0.8271, 0.1146, 0.0583]])"},"metadata":{}}]},{"cell_type":"code","source":"def get_sentiment_labels(aspect, tokenizer, model):\n    text = aspect['asp_with_context']\n    preds = predict_sentiment(text, tokenizer, model)\n    pred_dict = {'neutral': preds[0][0],\n                 'positive': preds[0][1],\n                 'negative': preds[0][2]}\n    pred_dict = {k: v for k, v in sorted(pred_dict.items(),\n                                         key=lambda item: item[1],\n                                         reverse=True)}\n    most_prob_label = list(pred_dict.items())[0][0]\n    highest_prob = list(pred_dict.items())[0][1]\n    if highest_prob > 0.65:\n        aspect['sentiment2'] = most_prob_label\n    else:\n        aspect['sentiment2'] = 'both'","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:45.281058Z","iopub.execute_input":"2021-12-27T12:34:45.281337Z","iopub.status.idle":"2021-12-27T12:34:45.288391Z","shell.execute_reply.started":"2021-12-27T12:34:45.281302Z","shell.execute_reply":"2021-12-27T12:34:45.287459Z"},"trusted":true},"execution_count":73,"outputs":[]},{"cell_type":"code","source":"for text in tqdm(new_text_aspects):\n    for sentence in text:\n        for aspect in sentence:\n            get_sentiment_labels(aspect, tokenizer, pretrained_model)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:34:45.289658Z","iopub.execute_input":"2021-12-27T12:34:45.290114Z","iopub.status.idle":"2021-12-27T12:36:42.429820Z","shell.execute_reply.started":"2021-12-27T12:34:45.290077Z","shell.execute_reply":"2021-12-27T12:36:42.429111Z"},"trusted":true},"execution_count":74,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/71 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"842007e25ead42fb83452d27184d5dd9"}},"metadata":{}}]},{"cell_type":"code","source":"new_text_aspects[0][2]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:42.431357Z","iopub.execute_input":"2021-12-27T12:36:42.431835Z","iopub.status.idle":"2021-12-27T12:36:42.439443Z","shell.execute_reply.started":"2021-12-27T12:36:42.431783Z","shell.execute_reply":"2021-12-27T12:36:42.438718Z"},"trusted":true},"execution_count":75,"outputs":[{"execution_count":75,"output_type":"execute_result","data":{"text/plain":"[{'asp_idx': 13823,\n  'asp_text': 'встретил',\n  'asp_start': 138,\n  'asp_end': 146,\n  'asp_with_context': 'встретил менеджер - темноволосая',\n  'sentiment2': 'neutral'},\n {'asp_idx': 13823,\n  'asp_text': 'менеджер',\n  'asp_start': 147,\n  'asp_end': 155,\n  'asp_with_context': 'менеджер - темноволосая стройная',\n  'sentiment2': 'neutral'},\n {'asp_idx': 13823,\n  'asp_text': 'девушка',\n  'asp_start': 179,\n  'asp_end': 186,\n  'asp_with_context': '- темноволосая стройная девушка , проводила к',\n  'sentiment2': 'neutral'},\n {'asp_idx': 13823,\n  'asp_text': 'проводила к столу',\n  'asp_start': 188,\n  'asp_end': 205,\n  'asp_with_context': 'стройная девушка , проводила к столу и дала меню',\n  'sentiment2': 'both'},\n {'asp_idx': 13823,\n  'asp_text': 'дала меню',\n  'asp_start': 208,\n  'asp_end': 217,\n  'asp_with_context': 'к столу и дала меню . .',\n  'sentiment2': 'negative'}]"},"metadata":{}}]},{"cell_type":"markdown","source":"Ощущение, что качество хуже, чем у предыдущего метода :(\n\n**Дальше происходит все то же самое, что было выше для другого метода.**","metadata":{}},{"cell_type":"code","source":"new_full_aspects_for_clf = []\nfor one_text_asps in new_text_aspects:\n    asps_for_clf = []\n    for one_sent_asps in one_text_asps:\n        asps_for_clf.append(prepare_for_classification(one_sent_asps))\n    new_full_aspects_for_clf.append(asps_for_clf)\n\nnew_all_pred_labels = []\nfor one_text in new_full_aspects_for_clf:\n    pred_labels = classify_aspects(one_text, svc, inv_mapper)\n    new_all_pred_labels.append(pred_labels)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:42.440853Z","iopub.execute_input":"2021-12-27T12:36:42.441361Z","iopub.status.idle":"2021-12-27T12:36:43.231611Z","shell.execute_reply.started":"2021-12-27T12:36:42.441325Z","shell.execute_reply":"2021-12-27T12:36:43.230871Z"},"trusted":true},"execution_count":76,"outputs":[]},{"cell_type":"code","source":"for i, text in enumerate(new_text_aspects):\n    for j, sentence in enumerate(text):\n        for n, aspect in enumerate(sentence):\n            aspect['category'] = new_all_pred_labels[i][j][n]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.233042Z","iopub.execute_input":"2021-12-27T12:36:43.233304Z","iopub.status.idle":"2021-12-27T12:36:43.239069Z","shell.execute_reply.started":"2021-12-27T12:36:43.233268Z","shell.execute_reply":"2021-12-27T12:36:43.238082Z"},"trusted":true},"execution_count":77,"outputs":[]},{"cell_type":"code","source":"new_text_aspects[0][2]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.240501Z","iopub.execute_input":"2021-12-27T12:36:43.241030Z","iopub.status.idle":"2021-12-27T12:36:43.254217Z","shell.execute_reply.started":"2021-12-27T12:36:43.240990Z","shell.execute_reply":"2021-12-27T12:36:43.253421Z"},"trusted":true},"execution_count":78,"outputs":[{"execution_count":78,"output_type":"execute_result","data":{"text/plain":"[{'asp_idx': 13823,\n  'asp_text': 'встретил',\n  'asp_start': 138,\n  'asp_end': 146,\n  'asp_with_context': 'встретил менеджер - темноволосая',\n  'sentiment2': 'neutral',\n  'category': 'Service'},\n {'asp_idx': 13823,\n  'asp_text': 'менеджер',\n  'asp_start': 147,\n  'asp_end': 155,\n  'asp_with_context': 'менеджер - темноволосая стройная',\n  'sentiment2': 'neutral',\n  'category': 'Service'},\n {'asp_idx': 13823,\n  'asp_text': 'девушка',\n  'asp_start': 179,\n  'asp_end': 186,\n  'asp_with_context': '- темноволосая стройная девушка , проводила к',\n  'sentiment2': 'neutral',\n  'category': 'Service'},\n {'asp_idx': 13823,\n  'asp_text': 'проводила к столу',\n  'asp_start': 188,\n  'asp_end': 205,\n  'asp_with_context': 'стройная девушка , проводила к столу и дала меню',\n  'sentiment2': 'both',\n  'category': 'Service'},\n {'asp_idx': 13823,\n  'asp_text': 'дала меню',\n  'asp_start': 208,\n  'asp_end': 217,\n  'asp_with_context': 'к столу и дала меню . .',\n  'sentiment2': 'negative',\n  'category': 'Service'}]"},"metadata":{}}]},{"cell_type":"code","source":"new_text_sentiments = {}\n\nfor text in new_text_aspects:\n\n    text_sent = {}\n    whole, interior, food, service, price = [], [], [], [], []\n    \n    for sentence in text:\n\n        for aspect in sentence:\n\n            text_idx = aspect['asp_idx']\n            categ = aspect['category']\n            sentim = aspect['sentiment2']\n            if categ == 'Whole':\n                whole.append(sentim)\n            elif categ == 'Interior':\n                interior.append(sentim)\n            elif categ == 'Food':\n                food.append(sentim)\n            elif categ == 'Service':\n                service.append(sentim)\n            elif categ == 'Price':\n                price.append(sentim)\n\n    food = Counter(food).most_common()\n    interior = Counter(interior).most_common()\n    price = Counter(price).most_common()\n    whole = Counter(whole).most_common()\n    service = Counter(service).most_common()\n\n    if len(food) == 0:\n        text_sent['Food'] = 'absence'\n    elif len(food) == 1:\n        text_sent['Food'] = food[0][0]\n    elif food[0][1] > food[1][1]:\n        text_sent['Food'] = food[0][0]\n    else:\n        text_sent['Food'] = 'both'\n\n    if len(interior) == 0:\n        text_sent['Interior'] = 'absence'\n    elif len(interior) == 1:\n        text_sent['Interior'] = interior[0][0]\n    elif interior[0][1] > interior[1][1]:\n        text_sent['Interior'] = interior[0][0]\n    else:\n        text_sent['Interior'] = 'both'\n\n    if len(price) == 0:\n        text_sent['Price'] = 'absence'\n    elif len(price) == 1:\n        text_sent['Price'] = price[0][0]\n    elif price[0][1] > price[1][1]:\n        text_sent['Price'] = price[0][0]\n    else:\n        text_sent['Price'] = 'both'\n\n    if len(whole) == 0:\n        text_sent['Whole'] = 'absence'\n    elif len(whole) == 1:\n        text_sent['Whole'] = whole[0][0]\n    elif whole[0][1] > whole[1][1]:\n        text_sent['Whole'] = whole[0][0]\n    else:\n        text_sent['Whole'] = 'both'\n\n    if len(service) == 0:\n        text_sent['Service'] = 'absence'\n    elif len(service) == 1:\n        text_sent['Service'] = service[0][0]\n    elif service[0][1] > service[1][1]:\n        text_sent['Service'] = service[0][0]\n    else:\n        text_sent['Service'] = 'both'\n\n    new_text_sentiments[text_idx] = text_sent","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.255817Z","iopub.execute_input":"2021-12-27T12:36:43.256062Z","iopub.status.idle":"2021-12-27T12:36:43.276259Z","shell.execute_reply.started":"2021-12-27T12:36:43.256030Z","shell.execute_reply":"2021-12-27T12:36:43.275448Z"},"trusted":true},"execution_count":79,"outputs":[]},{"cell_type":"code","source":"new_all_aspects_with_cats = []\nfor text in new_text_aspects:\n    for sent in text:\n        new_all_aspects_with_cats.extend(sent)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.277691Z","iopub.execute_input":"2021-12-27T12:36:43.277941Z","iopub.status.idle":"2021-12-27T12:36:43.286745Z","shell.execute_reply.started":"2021-12-27T12:36:43.277907Z","shell.execute_reply":"2021-12-27T12:36:43.285993Z"},"trusted":true},"execution_count":80,"outputs":[]},{"cell_type":"code","source":"new_all_aspects_with_cats[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.288057Z","iopub.execute_input":"2021-12-27T12:36:43.288450Z","iopub.status.idle":"2021-12-27T12:36:43.297625Z","shell.execute_reply.started":"2021-12-27T12:36:43.288413Z","shell.execute_reply":"2021-12-27T12:36:43.296971Z"},"trusted":true},"execution_count":81,"outputs":[{"execution_count":81,"output_type":"execute_result","data":{"text/plain":"{'asp_idx': 13823,\n 'asp_text': 'аппетит \"',\n 'asp_start': 8,\n 'asp_end': 16,\n 'asp_with_context': 'Зашли в \" аппетит \" случайно .',\n 'sentiment2': 'neutral',\n 'category': 'Whole'}"},"metadata":{}}]},{"cell_type":"code","source":"new_full_tagged_df = pd.DataFrame(new_all_aspects_with_cats)\nnew_full_tagged_df = new_full_tagged_df[['asp_idx', 'category', 'asp_text', 'asp_start',\n                                         'asp_end', 'sentiment2', 'asp_with_context']]\nnew_full_tagged_df.drop(columns=['asp_with_context'], inplace=True)\nnew_full_tagged_df.to_csv('new_sentiment_by_aspect.tsv', sep='\\t', index=False, header=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.298714Z","iopub.execute_input":"2021-12-27T12:36:43.299018Z","iopub.status.idle":"2021-12-27T12:36:43.318029Z","shell.execute_reply.started":"2021-12-27T12:36:43.298982Z","shell.execute_reply":"2021-12-27T12:36:43.317282Z"},"trusted":true},"execution_count":82,"outputs":[]},{"cell_type":"code","source":"new_categs_for_df = []\nfor text, sentiments in list(new_text_sentiments.items()):\n    for categ, sentim in list(sentiments.items()):\n        new_categs_for_df.append([text, categ, sentim])","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.319254Z","iopub.execute_input":"2021-12-27T12:36:43.319482Z","iopub.status.idle":"2021-12-27T12:36:43.325756Z","shell.execute_reply.started":"2021-12-27T12:36:43.319451Z","shell.execute_reply":"2021-12-27T12:36:43.324894Z"},"trusted":true},"execution_count":83,"outputs":[]},{"cell_type":"code","source":"new_categ_tagged_df = pd.DataFrame(new_categs_for_df, columns=['idx', 'category', 'sentiment'])\nnew_categ_tagged_df.to_csv('new_sentiment_by_category.tsv', sep='\\t', index=False, header=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-27T12:36:43.328186Z","iopub.execute_input":"2021-12-27T12:36:43.328635Z","iopub.status.idle":"2021-12-27T12:36:43.336580Z","shell.execute_reply.started":"2021-12-27T12:36:43.328602Z","shell.execute_reply":"2021-12-27T12:36:43.335929Z"},"trusted":true},"execution_count":84,"outputs":[]},{"cell_type":"markdown","source":"Ура! Оценка в другой тетрадке :)","metadata":{}}]}