Baseline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Загрузка данных"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Тренировочные данные состоят из полей 'text_id', 'text' и 11 полей с таргетами. Не стоит забывать, что может быть больше одной болезни для каждого случая."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Тестовые же данные содержат поля 'text_id' и 'text'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "      <th>эймериоз</th>\n",
       "      <th>абсцесс</th>\n",
       "      <th>диспепсия молодняка</th>\n",
       "      <th>остертагиоз</th>\n",
       "      <th>мастит</th>\n",
       "      <th>инфекционный ринотрахеит</th>\n",
       "      <th>отёк вымени</th>\n",
       "      <th>тенденит</th>\n",
       "      <th>сибирская язва</th>\n",
       "      <th>лишай</th>\n",
       "      <th>другое</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Корова, видимо вставая, раздавила себе сосок. ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Корове 8 лет! Месяц назад промеж четвертей вым...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_id                                               text  эймериоз  \\\n",
       "0        0  Корова, видимо вставая, раздавила себе сосок. ...         0   \n",
       "1        1  Корове 8 лет! Месяц назад промеж четвертей вым...         0   \n",
       "\n",
       "   абсцесс  диспепсия молодняка  остертагиоз  мастит  \\\n",
       "0        0                    0            0       1   \n",
       "1        0                    0            0       0   \n",
       "\n",
       "   инфекционный ринотрахеит  отёк вымени  тенденит  сибирская язва  лишай  \\\n",
       "0                         0            0         0               0      0   \n",
       "1                         0            0         0               1      1   \n",
       "\n",
       "   другое  \n",
       "0       0  \n",
       "1       1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>294</td>\n",
       "      <td>Понос у месячных телят. Подскажите методы и сп...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>295</td>\n",
       "      <td>Понос у телят, чем лечить? \\nЧем можно вылечит...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_id                                               text\n",
       "0      294  Понос у месячных телят. Подскажите методы и сп...\n",
       "1      295  Понос у телят, чем лечить? \\nЧем можно вылечит..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Базовая модель"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В качестве базового решения используется CatBoostClassifier, поддерживающий текстовые фичи. Помимо этого задача является мультилейбл классификацией, поэтому модель обернута в OneVsRestClassifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Делим данные на тренировочную и валидационную выборку**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(pd.DataFrame(train['text']), train[train.columns[2:]], test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Обучаем модель**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6747707\ttotal: 162ms\tremaining: 16.1s\n",
      "50:\tlearn: 0.3458487\ttotal: 1.46s\tremaining: 1.41s\n",
      "99:\tlearn: 0.2733018\ttotal: 2.58s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6751378\ttotal: 13.8ms\tremaining: 1.36s\n",
      "50:\tlearn: 0.3927736\ttotal: 1.32s\tremaining: 1.27s\n",
      "99:\tlearn: 0.3389888\ttotal: 2.58s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6741963\ttotal: 25.7ms\tremaining: 2.55s\n",
      "50:\tlearn: 0.3692726\ttotal: 1.29s\tremaining: 1.24s\n",
      "99:\tlearn: 0.3088863\ttotal: 2.53s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6721095\ttotal: 27.8ms\tremaining: 2.75s\n",
      "50:\tlearn: 0.3370063\ttotal: 1.44s\tremaining: 1.39s\n",
      "99:\tlearn: 0.2708549\ttotal: 2.69s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6703666\ttotal: 23.6ms\tremaining: 2.33s\n",
      "50:\tlearn: 0.3134060\ttotal: 1.37s\tremaining: 1.31s\n",
      "99:\tlearn: 0.2479107\ttotal: 2.6s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6698878\ttotal: 27.6ms\tremaining: 2.73s\n",
      "50:\tlearn: 0.3134208\ttotal: 1.21s\tremaining: 1.16s\n",
      "99:\tlearn: 0.2484044\ttotal: 2.41s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6715665\ttotal: 16ms\tremaining: 1.58s\n",
      "50:\tlearn: 0.2785950\ttotal: 1.27s\tremaining: 1.22s\n",
      "99:\tlearn: 0.2032067\ttotal: 2.52s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6703616\ttotal: 22.3ms\tremaining: 2.21s\n",
      "50:\tlearn: 0.3011587\ttotal: 1.28s\tremaining: 1.23s\n",
      "99:\tlearn: 0.2331644\ttotal: 2.51s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6720853\ttotal: 25.1ms\tremaining: 2.48s\n",
      "50:\tlearn: 0.3671037\ttotal: 1.37s\tremaining: 1.31s\n",
      "99:\tlearn: 0.3123558\ttotal: 2.63s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6705816\ttotal: 14.7ms\tremaining: 1.46s\n",
      "50:\tlearn: 0.3103462\ttotal: 1.2s\tremaining: 1.16s\n",
      "99:\tlearn: 0.2486406\ttotal: 2.4s\tremaining: 0us\n",
      "Learning rate set to 0.045854\n",
      "0:\tlearn: 0.6907435\ttotal: 24.1ms\tremaining: 2.38s\n",
      "50:\tlearn: 0.5984128\ttotal: 1.28s\tremaining: 1.23s\n",
      "99:\tlearn: 0.5449603\ttotal: 2.43s\tremaining: 0us\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=<catboost.core.CatBoostClassifier object at 0x0000022E6741EDC0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = OneVsRestClassifier(estimator=CatBoostClassifier(iterations = 100, text_features=['text'], \n",
    "                                                         verbose = 50, allow_writing_files=False))\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Считаем метрику"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В качестве \"gt\" (ground truth) функция принимает на вход датафрейм/массив из 10 столбцов (все классы, кроме \"другое\"). Предсказанные значения – \"pr\", также должны быть либо в виде датафрейма, либо в виде массива."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_loss_score(gt, pr):\n",
    "    \n",
    "    log_loss_ = 0\n",
    "    \n",
    "    gt = np.array(gt)\n",
    "    \n",
    "    for i in range(10):\n",
    "        log_loss_ += log_loss(gt[:, i], pr[:, i])\n",
    "        \n",
    "    return log_loss_ / 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3855734217281258"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss_score(y_val, model.predict_proba(X_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Создание файла отправки"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В файле с ответами должны быть вероятности для каждого класса. Сумма вероятностей в каждой строке может быть больше 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_columns = ['text_id'] + list(train.columns[2:-1])\n",
    "submission = pd.concat([test['text_id'], pd.DataFrame(model.predict_proba(pd.DataFrame(test['text']))[:, :10])], axis=1)\n",
    "submission.columns = submission_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>эймериоз</th>\n",
       "      <th>абсцесс</th>\n",
       "      <th>диспепсия молодняка</th>\n",
       "      <th>остертагиоз</th>\n",
       "      <th>мастит</th>\n",
       "      <th>инфекционный ринотрахеит</th>\n",
       "      <th>отёк вымени</th>\n",
       "      <th>тенденит</th>\n",
       "      <th>сибирская язва</th>\n",
       "      <th>лишай</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>294</td>\n",
       "      <td>0.158399</td>\n",
       "      <td>0.114065</td>\n",
       "      <td>0.121807</td>\n",
       "      <td>0.091488</td>\n",
       "      <td>0.087121</td>\n",
       "      <td>0.068731</td>\n",
       "      <td>0.047912</td>\n",
       "      <td>0.061100</td>\n",
       "      <td>0.096797</td>\n",
       "      <td>0.066527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>295</td>\n",
       "      <td>0.649903</td>\n",
       "      <td>0.130241</td>\n",
       "      <td>0.424610</td>\n",
       "      <td>0.385480</td>\n",
       "      <td>0.143775</td>\n",
       "      <td>0.102555</td>\n",
       "      <td>0.068045</td>\n",
       "      <td>0.271119</td>\n",
       "      <td>0.140103</td>\n",
       "      <td>0.101376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>296</td>\n",
       "      <td>0.497556</td>\n",
       "      <td>0.128907</td>\n",
       "      <td>0.417635</td>\n",
       "      <td>0.362003</td>\n",
       "      <td>0.085173</td>\n",
       "      <td>0.082718</td>\n",
       "      <td>0.053324</td>\n",
       "      <td>0.173252</td>\n",
       "      <td>0.108915</td>\n",
       "      <td>0.068368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>297</td>\n",
       "      <td>0.092075</td>\n",
       "      <td>0.166353</td>\n",
       "      <td>0.087311</td>\n",
       "      <td>0.084409</td>\n",
       "      <td>0.066975</td>\n",
       "      <td>0.075544</td>\n",
       "      <td>0.051250</td>\n",
       "      <td>0.067557</td>\n",
       "      <td>0.096998</td>\n",
       "      <td>0.066321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>298</td>\n",
       "      <td>0.099724</td>\n",
       "      <td>0.138278</td>\n",
       "      <td>0.103211</td>\n",
       "      <td>0.094123</td>\n",
       "      <td>0.186553</td>\n",
       "      <td>0.081351</td>\n",
       "      <td>0.088730</td>\n",
       "      <td>0.081809</td>\n",
       "      <td>0.114271</td>\n",
       "      <td>0.104003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_id  эймериоз   абсцесс  диспепсия молодняка  остертагиоз    мастит  \\\n",
       "0      294  0.158399  0.114065             0.121807     0.091488  0.087121   \n",
       "1      295  0.649903  0.130241             0.424610     0.385480  0.143775   \n",
       "2      296  0.497556  0.128907             0.417635     0.362003  0.085173   \n",
       "3      297  0.092075  0.166353             0.087311     0.084409  0.066975   \n",
       "4      298  0.099724  0.138278             0.103211     0.094123  0.186553   \n",
       "\n",
       "   инфекционный ринотрахеит  отёк вымени  тенденит  сибирская язва     лишай  \n",
       "0                  0.068731     0.047912  0.061100        0.096797  0.066527  \n",
       "1                  0.102555     0.068045  0.271119        0.140103  0.101376  \n",
       "2                  0.082718     0.053324  0.173252        0.108915  0.068368  \n",
       "3                  0.075544     0.051250  0.067557        0.096998  0.066321  \n",
       "4                  0.081351     0.088730  0.081809        0.114271  0.104003  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'span': [],\n",
       " 'label': [0.15839938785217741,\n",
       "  0.11406474371106279,\n",
       "  0.12180665939855911,\n",
       "  0.09148752585229543,\n",
       "  0.08712122855132899,\n",
       "  0.06873100350701257,\n",
       "  0.0479117525232059,\n",
       "  0.06110032039638013,\n",
       "  0.0967968198778296,\n",
       "  0.0665265225286165]}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_json = {str(k): {\"span\": list(), \"label\": list(v.values())} \\\n",
    "                   for k,v in submission.set_index('text_id').to_dict('index').items()}\n",
    "\n",
    "submission_json['294']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sample_submission.json', 'w') as final_submit:\n",
    "    json.dump(submission_json, final_submit, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}